Deduplication of Scholarly Documents using Locality Sensitive Hashing and Word Embeddings. Gyawali, B., Anastasiou, L., & Knoth, P. In Proceedings of the 12th Language Resources and Evaluation Conference, pages 901–910, Marseille, France, 2020. European Language Resources Association. ![Deduplication of Scholarly Documents using Locality Sensitive Hashing and Word Embeddings [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper abstract bibtex Deduplication is the task of identifying near and exact duplicate data items in a collection. In this paper, we present a novel method for deduplication of scholarly documents. We develop a hybrid model which uses structural similarity (locality sensitive hashing) and meaning representation (word embeddings) of document texts to determine (near) duplicates. Our collection constitutes a subset of multidisciplinary scholarly documents aggregated from research repositories. We identify several issues causing data inaccuracies in such collections and motivate the need for deduplication. In lack of existing dataset suitable for study of deduplication of scholarly documents, we create a ground truth dataset of $100K$ scholarly documents and conduct a series of experiments to empirically establish optimal values for the parameters of our deduplication method. Experimental evaluation shows that our method achieves a macro F1-score of 0.90. We productionise our method as a publicly accessible web API service serving deduplication of scholarly documents in real time.
@inproceedings{gyawali_deduplication_2020,
address = {Marseille, France},
title = {Deduplication of {Scholarly} {Documents} using {Locality} {Sensitive} {Hashing} and {Word} {Embeddings}},
isbn = {979-10-95546-34-4},
url = {https://aclanthology.org/2020.lrec-1.113},
abstract = {Deduplication is the task of identifying near and exact duplicate data items in a collection. In this paper, we present a novel method for deduplication of scholarly documents. We develop a hybrid model which uses structural similarity (locality sensitive hashing) and meaning representation (word embeddings) of document texts to determine (near) duplicates. Our collection constitutes a subset of multidisciplinary scholarly documents aggregated from research repositories. We identify several issues causing data inaccuracies in such collections and motivate the need for deduplication. In lack of existing dataset suitable for study of deduplication of scholarly documents, we create a ground truth dataset of \$100K\$ scholarly documents and conduct a series of experiments to empirically establish optimal values for the parameters of our deduplication method. Experimental evaluation shows that our method achieves a macro F1-score of 0.90. We productionise our method as a publicly accessible web API service serving deduplication of scholarly documents in real time.},
language = {English},
booktitle = {Proceedings of the 12th {Language} {Resources} and {Evaluation} {Conference}},
publisher = {European Language Resources Association},
author = {Gyawali, Bikash and Anastasiou, Lucas and Knoth, Petr},
year = {2020},
pages = {901--910},
}
Downloads: 0
{"_id":"5oE6EGMQNs24GsR8c","bibbaseid":"gyawali-anastasiou-knoth-deduplicationofscholarlydocumentsusinglocalitysensitivehashingandwordembeddings-2020","author_short":["Gyawali, B.","Anastasiou, L.","Knoth, P."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"Marseille, France","title":"Deduplication of Scholarly Documents using Locality Sensitive Hashing and Word Embeddings","isbn":"979-10-95546-34-4","url":"https://aclanthology.org/2020.lrec-1.113","abstract":"Deduplication is the task of identifying near and exact duplicate data items in a collection. In this paper, we present a novel method for deduplication of scholarly documents. We develop a hybrid model which uses structural similarity (locality sensitive hashing) and meaning representation (word embeddings) of document texts to determine (near) duplicates. Our collection constitutes a subset of multidisciplinary scholarly documents aggregated from research repositories. We identify several issues causing data inaccuracies in such collections and motivate the need for deduplication. In lack of existing dataset suitable for study of deduplication of scholarly documents, we create a ground truth dataset of $100K$ scholarly documents and conduct a series of experiments to empirically establish optimal values for the parameters of our deduplication method. Experimental evaluation shows that our method achieves a macro F1-score of 0.90. We productionise our method as a publicly accessible web API service serving deduplication of scholarly documents in real time.","language":"English","booktitle":"Proceedings of the 12th Language Resources and Evaluation Conference","publisher":"European Language Resources Association","author":[{"propositions":[],"lastnames":["Gyawali"],"firstnames":["Bikash"],"suffixes":[]},{"propositions":[],"lastnames":["Anastasiou"],"firstnames":["Lucas"],"suffixes":[]},{"propositions":[],"lastnames":["Knoth"],"firstnames":["Petr"],"suffixes":[]}],"year":"2020","pages":"901–910","bibtex":"@inproceedings{gyawali_deduplication_2020,\n\taddress = {Marseille, France},\n\ttitle = {Deduplication of {Scholarly} {Documents} using {Locality} {Sensitive} {Hashing} and {Word} {Embeddings}},\n\tisbn = {979-10-95546-34-4},\n\turl = {https://aclanthology.org/2020.lrec-1.113},\n\tabstract = {Deduplication is the task of identifying near and exact duplicate data items in a collection. In this paper, we present a novel method for deduplication of scholarly documents. We develop a hybrid model which uses structural similarity (locality sensitive hashing) and meaning representation (word embeddings) of document texts to determine (near) duplicates. Our collection constitutes a subset of multidisciplinary scholarly documents aggregated from research repositories. We identify several issues causing data inaccuracies in such collections and motivate the need for deduplication. In lack of existing dataset suitable for study of deduplication of scholarly documents, we create a ground truth dataset of \\$100K\\$ scholarly documents and conduct a series of experiments to empirically establish optimal values for the parameters of our deduplication method. Experimental evaluation shows that our method achieves a macro F1-score of 0.90. We productionise our method as a publicly accessible web API service serving deduplication of scholarly documents in real time.},\n\tlanguage = {English},\n\tbooktitle = {Proceedings of the 12th {Language} {Resources} and {Evaluation} {Conference}},\n\tpublisher = {European Language Resources Association},\n\tauthor = {Gyawali, Bikash and Anastasiou, Lucas and Knoth, Petr},\n\tyear = {2020},\n\tpages = {901--910},\n}\n\n","author_short":["Gyawali, B.","Anastasiou, L.","Knoth, P."],"key":"gyawali_deduplication_2020","id":"gyawali_deduplication_2020","bibbaseid":"gyawali-anastasiou-knoth-deduplicationofscholarlydocumentsusinglocalitysensitivehashingandwordembeddings-2020","role":"author","urls":{"Paper":"https://aclanthology.org/2020.lrec-1.113"},"metadata":{"authorlinks":{}}},"bibtype":"inproceedings","biburl":"https://api.zotero.org/groups/2532329/items?key=3gRHj8hfOfGJBww9EhlzpL4j&format=bibtex&limit=100","dataSources":["72w8hmKFm76DkeXyN","RoBpvtgReM45iQXQm","fQ66AuPM43rBsaruw"],"keywords":[],"search_terms":["deduplication","scholarly","documents","using","locality","sensitive","hashing","word","embeddings","gyawali","anastasiou","knoth"],"title":"Deduplication of Scholarly Documents using Locality Sensitive Hashing and Word Embeddings","year":2020}