Lightweight Adaptation of Neural Language Models via Subspace Embedding. Jaiswal, A. K. & Liu, H. In Proceedings of the 32nd ACM International Conference on Information and Knowledge Management, of CIKM '23, pages 3968–3972, New York, NY, USA, October, 2023. Association for Computing Machinery. 0 citations (Semantic Scholar/DOI) [2024-01-12]![Lightweight Adaptation of Neural Language Models via Subspace Embedding [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper doi abstract bibtex Traditional neural word embeddings are usually dependent on a richer diversity of vocabulary. However, the language models recline to cover major vocabularies via the word embedding parameters, in particular, for multilingual language models that generally cover a significant part of their overall learning parameters. In this work, we present a new compact embedding structure to reduce the memory footprint of the pre-trained language models with a sacrifice of up to 4% absolute accuracy. The embeddings vectors reconstruction follows a set of subspace embeddings and an assignment procedure via the contextual relationship among tokens from pre-trained language models. The subspace embedding structure1 calibrates to masked language models, to evaluate our compact embedding structure on similarity and textual entailment tasks, sentence and paraphrase tasks. Our experimental evaluation shows that the subspace embeddings achieve compression rates beyond 99.8% in comparison with the original embeddings for the language models on XNLI and GLUE benchmark suites.
@inproceedings{jaiswal_lightweight_2023,
address = {New York, NY, USA},
series = {{CIKM} '23},
title = {Lightweight {Adaptation} of {Neural} {Language} {Models} via {Subspace} {Embedding}},
isbn = {9798400701245},
url = {https://dl.acm.org/doi/10.1145/3583780.3615269},
doi = {10.1145/3583780.3615269},
abstract = {Traditional neural word embeddings are usually dependent on a richer diversity of vocabulary. However, the language models recline to cover major vocabularies via the word embedding parameters, in particular, for multilingual language models that generally cover a significant part of their overall learning parameters. In this work, we present a new compact embedding structure to reduce the memory footprint of the pre-trained language models with a sacrifice of up to 4\% absolute accuracy. The embeddings vectors reconstruction follows a set of subspace embeddings and an assignment procedure via the contextual relationship among tokens from pre-trained language models. The subspace embedding structure1 calibrates to masked language models, to evaluate our compact embedding structure on similarity and textual entailment tasks, sentence and paraphrase tasks. Our experimental evaluation shows that the subspace embeddings achieve compression rates beyond 99.8\% in comparison with the original embeddings for the language models on XNLI and GLUE benchmark suites.},
urldate = {2024-01-12},
booktitle = {Proceedings of the 32nd {ACM} {International} {Conference} on {Information} and {Knowledge} {Management}},
publisher = {Association for Computing Machinery},
author = {Jaiswal, Amit Kumar and Liu, Haiming},
month = oct,
year = {2023},
note = {0 citations (Semantic Scholar/DOI) [2024-01-12]},
keywords = {language model, natural language understanding, word embedding},
pages = {3968--3972},
}
Downloads: 0
{"_id":"qyS4DM8p3DQgpYd7J","bibbaseid":"jaiswal-liu-lightweightadaptationofneurallanguagemodelsviasubspaceembedding-2023","author_short":["Jaiswal, A. K.","Liu, H."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"New York, NY, USA","series":"CIKM '23","title":"Lightweight Adaptation of Neural Language Models via Subspace Embedding","isbn":"9798400701245","url":"https://dl.acm.org/doi/10.1145/3583780.3615269","doi":"10.1145/3583780.3615269","abstract":"Traditional neural word embeddings are usually dependent on a richer diversity of vocabulary. However, the language models recline to cover major vocabularies via the word embedding parameters, in particular, for multilingual language models that generally cover a significant part of their overall learning parameters. In this work, we present a new compact embedding structure to reduce the memory footprint of the pre-trained language models with a sacrifice of up to 4% absolute accuracy. The embeddings vectors reconstruction follows a set of subspace embeddings and an assignment procedure via the contextual relationship among tokens from pre-trained language models. The subspace embedding structure1 calibrates to masked language models, to evaluate our compact embedding structure on similarity and textual entailment tasks, sentence and paraphrase tasks. Our experimental evaluation shows that the subspace embeddings achieve compression rates beyond 99.8% in comparison with the original embeddings for the language models on XNLI and GLUE benchmark suites.","urldate":"2024-01-12","booktitle":"Proceedings of the 32nd ACM International Conference on Information and Knowledge Management","publisher":"Association for Computing Machinery","author":[{"propositions":[],"lastnames":["Jaiswal"],"firstnames":["Amit","Kumar"],"suffixes":[]},{"propositions":[],"lastnames":["Liu"],"firstnames":["Haiming"],"suffixes":[]}],"month":"October","year":"2023","note":"0 citations (Semantic Scholar/DOI) [2024-01-12]","keywords":"language model, natural language understanding, word embedding","pages":"3968–3972","bibtex":"@inproceedings{jaiswal_lightweight_2023,\n\taddress = {New York, NY, USA},\n\tseries = {{CIKM} '23},\n\ttitle = {Lightweight {Adaptation} of {Neural} {Language} {Models} via {Subspace} {Embedding}},\n\tisbn = {9798400701245},\n\turl = {https://dl.acm.org/doi/10.1145/3583780.3615269},\n\tdoi = {10.1145/3583780.3615269},\n\tabstract = {Traditional neural word embeddings are usually dependent on a richer diversity of vocabulary. However, the language models recline to cover major vocabularies via the word embedding parameters, in particular, for multilingual language models that generally cover a significant part of their overall learning parameters. In this work, we present a new compact embedding structure to reduce the memory footprint of the pre-trained language models with a sacrifice of up to 4\\% absolute accuracy. The embeddings vectors reconstruction follows a set of subspace embeddings and an assignment procedure via the contextual relationship among tokens from pre-trained language models. The subspace embedding structure1 calibrates to masked language models, to evaluate our compact embedding structure on similarity and textual entailment tasks, sentence and paraphrase tasks. Our experimental evaluation shows that the subspace embeddings achieve compression rates beyond 99.8\\% in comparison with the original embeddings for the language models on XNLI and GLUE benchmark suites.},\n\turldate = {2024-01-12},\n\tbooktitle = {Proceedings of the 32nd {ACM} {International} {Conference} on {Information} and {Knowledge} {Management}},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Jaiswal, Amit Kumar and Liu, Haiming},\n\tmonth = oct,\n\tyear = {2023},\n\tnote = {0 citations (Semantic Scholar/DOI) [2024-01-12]},\n\tkeywords = {language model, natural language understanding, word embedding},\n\tpages = {3968--3972},\n}\n\n","author_short":["Jaiswal, A. K.","Liu, H."],"key":"jaiswal_lightweight_2023","id":"jaiswal_lightweight_2023","bibbaseid":"jaiswal-liu-lightweightadaptationofneurallanguagemodelsviasubspaceembedding-2023","role":"author","urls":{"Paper":"https://dl.acm.org/doi/10.1145/3583780.3615269"},"keyword":["language model","natural language understanding","word embedding"],"metadata":{"authorlinks":{}},"html":""},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero/ifromm","dataSources":["N4kJAiLiJ7kxfNsoh"],"keywords":["language model","natural language understanding","word embedding"],"search_terms":["lightweight","adaptation","neural","language","models","via","subspace","embedding","jaiswal","liu"],"title":"Lightweight Adaptation of Neural Language Models via Subspace Embedding","year":2023}