PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification

PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification. Yang, Y., Zhang, Y., Tar, C., & Baldridge, J. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pages 3685–3690, Hong Kong, China, 2019. Association for Computational Linguistics.

Paper doi abstract bibtex

Most existing work on adversarial data generation focuses on English. For example, PAWS (Paraphrase Adversaries from Word Scrambling) consists of challenging English paraphrase identification pairs from Wikipedia and Quora. We remedy this gap with PAWS-X, a new dataset of 23,659 human translated PAWS evaluation pairs in six typologically distinct languages: French, Spanish, German, Chinese, Japanese, and Korean. We provide baseline numbers for three models with different capacity to capture non-local context and sentence structure, and using different multilingual training and evaluation regimes. Multilingual BERT fine-tuned on PAWS English plus machine-translated data performs the best, with a range of 83.1-90.8 accuracy across the non-English languages and an average accuracy gain of 23% over the next best model. PAWS-X shows the effectiveness of deep, multilingual pre-training while also leaving considerable headroom as a new challenge to drive multilingual research that better captures structure and contextual information.

@inproceedings{yang_paws-x_2019,
	address = {Hong Kong, China},
	title = {{PAWS}-{X}: {A} {Cross}-lingual {Adversarial} {Dataset} for {Paraphrase} {Identification}},
	shorttitle = {{PAWS}-{X}},
	url = {https://www.aclweb.org/anthology/D19-1382},
	doi = {10.18653/v1/D19-1382},
	abstract = {Most existing work on adversarial data generation focuses on English. For example, PAWS (Paraphrase Adversaries from Word Scrambling) consists of challenging English paraphrase identification pairs from Wikipedia and Quora. We remedy this gap with PAWS-X, a new dataset of 23,659 human translated PAWS evaluation pairs in six typologically distinct languages: French, Spanish, German, Chinese, Japanese, and Korean. We provide baseline numbers for three models with different capacity to capture non-local context and sentence structure, and using different multilingual training and evaluation regimes. Multilingual BERT fine-tuned on PAWS English plus machine-translated data performs the best, with a range of 83.1-90.8 accuracy across the non-English languages and an average accuracy gain of 23\% over the next best model. PAWS-X shows the effectiveness of deep, multilingual pre-training while also leaving considerable headroom as a new challenge to drive multilingual research that better captures structure and contextual information.},
	language = {en},
	urldate = {2025-03-12},
	booktitle = {Proceedings of the 2019 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} and the 9th {International} {Joint} {Conference} on {Natural} {Language} {Processing} ({EMNLP}-{IJCNLP})},
	publisher = {Association for Computational Linguistics},
	author = {Yang, Yinfei and Zhang, Yuan and Tar, Chris and Baldridge, Jason},
	year = {2019},
	pages = {3685--3690},
}

Downloads: 0

{"_id":"cqsgvFeTEH7pZjQRf","bibbaseid":"yang-zhang-tar-baldridge-pawsxacrosslingualadversarialdatasetforparaphraseidentification-2019","author_short":["Yang, Y.","Zhang, Y.","Tar, C.","Baldridge, J."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"Hong Kong, China","title":"PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification","shorttitle":"PAWS-X","url":"https://www.aclweb.org/anthology/D19-1382","doi":"10.18653/v1/D19-1382","abstract":"Most existing work on adversarial data generation focuses on English. For example, PAWS (Paraphrase Adversaries from Word Scrambling) consists of challenging English paraphrase identification pairs from Wikipedia and Quora. We remedy this gap with PAWS-X, a new dataset of 23,659 human translated PAWS evaluation pairs in six typologically distinct languages: French, Spanish, German, Chinese, Japanese, and Korean. We provide baseline numbers for three models with different capacity to capture non-local context and sentence structure, and using different multilingual training and evaluation regimes. Multilingual BERT fine-tuned on PAWS English plus machine-translated data performs the best, with a range of 83.1-90.8 accuracy across the non-English languages and an average accuracy gain of 23% over the next best model. PAWS-X shows the effectiveness of deep, multilingual pre-training while also leaving considerable headroom as a new challenge to drive multilingual research that better captures structure and contextual information.","language":"en","urldate":"2025-03-12","booktitle":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","publisher":"Association for Computational Linguistics","author":[{"propositions":[],"lastnames":["Yang"],"firstnames":["Yinfei"],"suffixes":[]},{"propositions":[],"lastnames":["Zhang"],"firstnames":["Yuan"],"suffixes":[]},{"propositions":[],"lastnames":["Tar"],"firstnames":["Chris"],"suffixes":[]},{"propositions":[],"lastnames":["Baldridge"],"firstnames":["Jason"],"suffixes":[]}],"year":"2019","pages":"3685–3690","bibtex":"@inproceedings{yang_paws-x_2019,\n\taddress = {Hong Kong, China},\n\ttitle = {{PAWS}-{X}: {A} {Cross}-lingual {Adversarial} {Dataset} for {Paraphrase} {Identification}},\n\tshorttitle = {{PAWS}-{X}},\n\turl = {https://www.aclweb.org/anthology/D19-1382},\n\tdoi = {10.18653/v1/D19-1382},\n\tabstract = {Most existing work on adversarial data generation focuses on English. For example, PAWS (Paraphrase Adversaries from Word Scrambling) consists of challenging English paraphrase identification pairs from Wikipedia and Quora. We remedy this gap with PAWS-X, a new dataset of 23,659 human translated PAWS evaluation pairs in six typologically distinct languages: French, Spanish, German, Chinese, Japanese, and Korean. We provide baseline numbers for three models with different capacity to capture non-local context and sentence structure, and using different multilingual training and evaluation regimes. Multilingual BERT fine-tuned on PAWS English plus machine-translated data performs the best, with a range of 83.1-90.8 accuracy across the non-English languages and an average accuracy gain of 23\\% over the next best model. PAWS-X shows the effectiveness of deep, multilingual pre-training while also leaving considerable headroom as a new challenge to drive multilingual research that better captures structure and contextual information.},\n\tlanguage = {en},\n\turldate = {2025-03-12},\n\tbooktitle = {Proceedings of the 2019 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} and the 9th {International} {Joint} {Conference} on {Natural} {Language} {Processing} ({EMNLP}-{IJCNLP})},\n\tpublisher = {Association for Computational Linguistics},\n\tauthor = {Yang, Yinfei and Zhang, Yuan and Tar, Chris and Baldridge, Jason},\n\tyear = {2019},\n\tpages = {3685--3690},\n}\n\n\n\n","author_short":["Yang, Y.","Zhang, Y.","Tar, C.","Baldridge, J."],"key":"yang_paws-x_2019","id":"yang_paws-x_2019","bibbaseid":"yang-zhang-tar-baldridge-pawsxacrosslingualadversarialdatasetforparaphraseidentification-2019","role":"author","urls":{"Paper":"https://www.aclweb.org/anthology/D19-1382"},"metadata":{"authorlinks":{}}},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero/abhishek-p","dataSources":["h7kKWXpJh2iaX92T5"],"keywords":[],"search_terms":["paws","cross","lingual","adversarial","dataset","paraphrase","identification","yang","zhang","tar","baldridge"],"title":"PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification","year":2019}