@article{
 title = {XLNet: Generalized Autoregressive Pretraining for Language Understanding},
 type = {article},
 year = {2019},
 volume = {32},
 websites = {https://arxiv.org/abs/1906.08237v2},
 month = {6},
 publisher = {Neural information processing systems foundation},
 day = {19},
 id = {ed18f9c6-a45b-38ef-9dd5-2538d69b3220},
 created = {2021-08-24T07:17:41.452Z},
 accessed = {2021-08-24},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-08-24T07:17:44.148Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {c509f25c-b687-4ab5-8859-72131b6658d3},
 private_publication = {false},
 abstract = {With the capability of modeling bidirectional contexts, denoising
autoencoding based pretraining like BERT achieves better performance than
pretraining approaches based on autoregressive language modeling. However,
relying on corrupting the input with masks, BERT neglects dependency between
the masked positions and suffers from a pretrain-finetune discrepancy. In light
of these pros and cons, we propose XLNet, a generalized autoregressive
pretraining method that (1) enables learning bidirectional contexts by
maximizing the expected likelihood over all permutations of the factorization
order and (2) overcomes the limitations of BERT thanks to its autoregressive
formulation. Furthermore, XLNet integrates ideas from Transformer-XL, the
state-of-the-art autoregressive model, into pretraining. Empirically, under
comparable experiment settings, XLNet outperforms BERT on 20 tasks, often by a
large margin, including question answering, natural language inference,
sentiment analysis, and document ranking.},
 bibtype = {article},
 author = {Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Ruslan and Le, Quoc V.},
 journal = {Advances in Neural Information Processing Systems}
}

{"_id":"LdBJkjaiK992mvWz7","bibbaseid":"yang-dai-yang-carbonell-salakhutdinov-le-xlnetgeneralizedautoregressivepretrainingforlanguageunderstanding-2019","authorIDs":[],"author_short":["Yang, Z.","Dai, Z.","Yang, Y.","Carbonell, J.","Salakhutdinov, R.","Le, Q., V."],"bibdata":{"title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding","type":"article","year":"2019","volume":"32","websites":"https://arxiv.org/abs/1906.08237v2","month":"6","publisher":"Neural information processing systems foundation","day":"19","id":"ed18f9c6-a45b-38ef-9dd5-2538d69b3220","created":"2021-08-24T07:17:41.452Z","accessed":"2021-08-24","file_attached":"true","profile_id":"48fc0258-023d-3602-860e-824092d62c56","group_id":"1ff583c0-be37-34fa-9c04-73c69437d354","last_modified":"2021-08-24T07:17:44.148Z","read":false,"starred":false,"authored":false,"confirmed":false,"hidden":false,"folder_uuids":"c509f25c-b687-4ab5-8859-72131b6658d3","private_publication":false,"abstract":"With the capability of modeling bidirectional contexts, denoising\nautoencoding based pretraining like BERT achieves better performance than\npretraining approaches based on autoregressive language modeling. However,\nrelying on corrupting the input with masks, BERT neglects dependency between\nthe masked positions and suffers from a pretrain-finetune discrepancy. In light\nof these pros and cons, we propose XLNet, a generalized autoregressive\npretraining method that (1) enables learning bidirectional contexts by\nmaximizing the expected likelihood over all permutations of the factorization\norder and (2) overcomes the limitations of BERT thanks to its autoregressive\nformulation. Furthermore, XLNet integrates ideas from Transformer-XL, the\nstate-of-the-art autoregressive model, into pretraining. Empirically, under\ncomparable experiment settings, XLNet outperforms BERT on 20 tasks, often by a\nlarge margin, including question answering, natural language inference,\nsentiment analysis, and document ranking.","bibtype":"article","author":"Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Ruslan and Le, Quoc V.","journal":"Advances in Neural Information Processing Systems","bibtex":"@article{\n title = {XLNet: Generalized Autoregressive Pretraining for Language Understanding},\n type = {article},\n year = {2019},\n volume = {32},\n websites = {https://arxiv.org/abs/1906.08237v2},\n month = {6},\n publisher = {Neural information processing systems foundation},\n day = {19},\n id = {ed18f9c6-a45b-38ef-9dd5-2538d69b3220},\n created = {2021-08-24T07:17:41.452Z},\n accessed = {2021-08-24},\n file_attached = {true},\n profile_id = {48fc0258-023d-3602-860e-824092d62c56},\n group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},\n last_modified = {2021-08-24T07:17:44.148Z},\n read = {false},\n starred = {false},\n authored = {false},\n confirmed = {false},\n hidden = {false},\n folder_uuids = {c509f25c-b687-4ab5-8859-72131b6658d3},\n private_publication = {false},\n abstract = {With the capability of modeling bidirectional contexts, denoising\nautoencoding based pretraining like BERT achieves better performance than\npretraining approaches based on autoregressive language modeling. However,\nrelying on corrupting the input with masks, BERT neglects dependency between\nthe masked positions and suffers from a pretrain-finetune discrepancy. In light\nof these pros and cons, we propose XLNet, a generalized autoregressive\npretraining method that (1) enables learning bidirectional contexts by\nmaximizing the expected likelihood over all permutations of the factorization\norder and (2) overcomes the limitations of BERT thanks to its autoregressive\nformulation. Furthermore, XLNet integrates ideas from Transformer-XL, the\nstate-of-the-art autoregressive model, into pretraining. Empirically, under\ncomparable experiment settings, XLNet outperforms BERT on 20 tasks, often by a\nlarge margin, including question answering, natural language inference,\nsentiment analysis, and document ranking.},\n bibtype = {article},\n author = {Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Ruslan and Le, Quoc V.},\n journal = {Advances in Neural Information Processing Systems}\n}","author_short":["Yang, Z.","Dai, Z.","Yang, Y.","Carbonell, J.","Salakhutdinov, R.","Le, Q., V."],"urls":{"Paper":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c/file/229aeea5-7751-f171-3849-5e6af01386ce/full_text.pdf.pdf","Website":"https://arxiv.org/abs/1906.08237v2"},"biburl":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c","bibbaseid":"yang-dai-yang-carbonell-salakhutdinov-le-xlnetgeneralizedautoregressivepretrainingforlanguageunderstanding-2019","role":"author","metadata":{"authorlinks":{}},"downloads":0},"bibtype":"article","biburl":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c","creationDate":"2020-01-05T22:03:38.496Z","downloads":0,"keywords":[],"search_terms":["xlnet","generalized","autoregressive","pretraining","language","understanding","yang","dai","yang","carbonell","salakhutdinov","le"],"title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding","year":2019,"dataSources":["okYcdTpf4JJ2zkj7A","QGwcHf7xnb5mCCQi7","ya2CyA73rpZseyrZ8","SYqcwHeTFu9kg8TXh","2252seNhipfTmjEBQ","znj7izS5PeehdLR3G"]}