Unsupervised Domain Adaptation of Contextualized Embeddings: A Case Study in Early Modern English

Unsupervised Domain Adaptation of Contextualized Embeddings: A Case Study in Early Modern English. Han, X. & Eisenstein, J.

Paper abstract bibtex

Contextualized word embeddings such as ELMo and BERT provide a foundation for strong performance across a range of natural language processing tasks, in part by pretraining on a large and topically-diverse corpus. However, the applicability of this approach is unknown when the target domain varies substantially from the text used during pretraining. Specifically, we are interested the scenario in which labeled data is available in only a canonical source domain such as newstext, and the target domain is distinct from both the labeled corpus and the pretraining data. To address this scenario, we propose domain-adaptive fine-tuning, in which the contextualized embeddings are adapted by masked language modeling on the target domain. We test this approach on the challenging domain of Early Modern English, which differs substantially from existing pretraining corpora. Domain-adaptive fine-tuning yields an improvement of 4\textbackslash% in part-of-speech tagging accuracy over a BERT baseline, substantially improving on prior work on this task.

@article{hanUnsupervisedDomainAdaptation2019,
  archivePrefix = {arXiv},
  eprinttype = {arxiv},
  eprint = {1904.02817},
  primaryClass = {cs},
  title = {Unsupervised {{Domain Adaptation}} of {{Contextualized Embeddings}}: {{A Case Study}} in {{Early Modern English}}},
  url = {http://arxiv.org/abs/1904.02817},
  shorttitle = {Unsupervised {{Domain Adaptation}} of {{Contextualized Embeddings}}},
  abstract = {Contextualized word embeddings such as ELMo and BERT provide a foundation for strong performance across a range of natural language processing tasks, in part by pretraining on a large and topically-diverse corpus. However, the applicability of this approach is unknown when the target domain varies substantially from the text used during pretraining. Specifically, we are interested the scenario in which labeled data is available in only a canonical source domain such as newstext, and the target domain is distinct from both the labeled corpus and the pretraining data. To address this scenario, we propose domain-adaptive fine-tuning, in which the contextualized embeddings are adapted by masked language modeling on the target domain. We test this approach on the challenging domain of Early Modern English, which differs substantially from existing pretraining corpora. Domain-adaptive fine-tuning yields an improvement of 4\textbackslash\% in part-of-speech tagging accuracy over a BERT baseline, substantially improving on prior work on this task.},
  urldate = {2019-04-08},
  date = {2019-04-04},
  keywords = {Computer Science - Digital Libraries,Computer Science - Computation and Language,Computer Science - Machine Learning},
  author = {Han, Xiaochuang and Eisenstein, Jacob},
  file = {/home/dimitri/Nextcloud/Zotero/storage/RT8FLXP4/Han and Eisenstein - 2019 - Unsupervised Domain Adaptation of Contextualized E.pdf;/home/dimitri/Nextcloud/Zotero/storage/4B9LSUQF/1904.html}
}

Downloads: 0

{"_id":"b7946fp2RweFSYEB6","bibbaseid":"han-eisenstein-unsuperviseddomainadaptationofcontextualizedembeddingsacasestudyinearlymodernenglish","authorIDs":[],"author_short":["Han, X.","Eisenstein, J."],"bibdata":{"bibtype":"article","type":"article","archiveprefix":"arXiv","eprinttype":"arxiv","eprint":"1904.02817","primaryclass":"cs","title":"Unsupervised Domain Adaptation of Contextualized Embeddings: A Case Study in Early Modern English","url":"http://arxiv.org/abs/1904.02817","shorttitle":"Unsupervised Domain Adaptation of Contextualized Embeddings","abstract":"Contextualized word embeddings such as ELMo and BERT provide a foundation for strong performance across a range of natural language processing tasks, in part by pretraining on a large and topically-diverse corpus. However, the applicability of this approach is unknown when the target domain varies substantially from the text used during pretraining. Specifically, we are interested the scenario in which labeled data is available in only a canonical source domain such as newstext, and the target domain is distinct from both the labeled corpus and the pretraining data. To address this scenario, we propose domain-adaptive fine-tuning, in which the contextualized embeddings are adapted by masked language modeling on the target domain. We test this approach on the challenging domain of Early Modern English, which differs substantially from existing pretraining corpora. Domain-adaptive fine-tuning yields an improvement of 4\\textbackslash% in part-of-speech tagging accuracy over a BERT baseline, substantially improving on prior work on this task.","urldate":"2019-04-08","date":"2019-04-04","keywords":"Computer Science - Digital Libraries,Computer Science - Computation and Language,Computer Science - Machine Learning","author":[{"propositions":[],"lastnames":["Han"],"firstnames":["Xiaochuang"],"suffixes":[]},{"propositions":[],"lastnames":["Eisenstein"],"firstnames":["Jacob"],"suffixes":[]}],"file":"/home/dimitri/Nextcloud/Zotero/storage/RT8FLXP4/Han and Eisenstein - 2019 - Unsupervised Domain Adaptation of Contextualized E.pdf;/home/dimitri/Nextcloud/Zotero/storage/4B9LSUQF/1904.html","bibtex":"@article{hanUnsupervisedDomainAdaptation2019,\n archivePrefix = {arXiv},\n eprinttype = {arxiv},\n eprint = {1904.02817},\n primaryClass = {cs},\n title = {Unsupervised {{Domain Adaptation}} of {{Contextualized Embeddings}}: {{A Case Study}} in {{Early Modern English}}},\n url = {http://arxiv.org/abs/1904.02817},\n shorttitle = {Unsupervised {{Domain Adaptation}} of {{Contextualized Embeddings}}},\n abstract = {Contextualized word embeddings such as ELMo and BERT provide a foundation for strong performance across a range of natural language processing tasks, in part by pretraining on a large and topically-diverse corpus. However, the applicability of this approach is unknown when the target domain varies substantially from the text used during pretraining. Specifically, we are interested the scenario in which labeled data is available in only a canonical source domain such as newstext, and the target domain is distinct from both the labeled corpus and the pretraining data. To address this scenario, we propose domain-adaptive fine-tuning, in which the contextualized embeddings are adapted by masked language modeling on the target domain. We test this approach on the challenging domain of Early Modern English, which differs substantially from existing pretraining corpora. Domain-adaptive fine-tuning yields an improvement of 4\\textbackslash\\% in part-of-speech tagging accuracy over a BERT baseline, substantially improving on prior work on this task.},\n urldate = {2019-04-08},\n date = {2019-04-04},\n keywords = {Computer Science - Digital Libraries,Computer Science - Computation and Language,Computer Science - Machine Learning},\n author = {Han, Xiaochuang and Eisenstein, Jacob},\n file = {/home/dimitri/Nextcloud/Zotero/storage/RT8FLXP4/Han and Eisenstein - 2019 - Unsupervised Domain Adaptation of Contextualized E.pdf;/home/dimitri/Nextcloud/Zotero/storage/4B9LSUQF/1904.html}\n}\n\n","author_short":["Han, X.","Eisenstein, J."],"key":"hanUnsupervisedDomainAdaptation2019","id":"hanUnsupervisedDomainAdaptation2019","bibbaseid":"han-eisenstein-unsuperviseddomainadaptationofcontextualizedembeddingsacasestudyinearlymodernenglish","role":"author","urls":{"Paper":"http://arxiv.org/abs/1904.02817"},"keyword":["Computer Science - Digital Libraries","Computer Science - Computation and Language","Computer Science - Machine Learning"],"downloads":0},"bibtype":"article","biburl":"https://raw.githubusercontent.com/dlozeve/newblog/master/bib/all.bib","creationDate":"2020-01-08T20:39:39.318Z","downloads":0,"keywords":["computer science - digital libraries","computer science - computation and language","computer science - machine learning"],"search_terms":["unsupervised","domain","adaptation","contextualized","embeddings","case","study","early","modern","english","han","eisenstein"],"title":"Unsupervised Domain Adaptation of Contextualized Embeddings: A Case Study in Early Modern English","year":null,"dataSources":["3XqdvqRE7zuX4cm8m"]}