Is GPT fit for KGQA? – Preliminary results. Klager, G. G. & Polleres, A. In Proceedings of the International Workshop on Knowledge Graph Generation from Text (Text2KG2023), co-located with Extended Semantic Web Conference 2023 (ESWC 2023), May, 2023. to appear![Is GPT fit for KGQA? – Preliminary results [pdf]](https://bibbase.org/img/filetypes/pdf.svg "pdf")
Paper abstract bibtex In this paper we report about preliminary results on running question answering benchmarks against the recently hyped conversational AI services such as ChatGPT: we focus on questions that are known to be possible to be answered by information in existing Knowledge graphs such as Wikidata. In a preliminary study we experiment, on the one hand, with questions from established KGQA benchmarks, and on the other hand, present a set of questions established in a student experiment, which should be particularly hard for Large Language Models (LLMs) to answer, mainly focusing on questions on recent events. In a second experiment, we assess how far GPT could be used for query generation in SPARQL. While our results are mostly negative for now, we hope to provide insights for further research in this direction, in terms of isolating and discussing the most obvious challenges and gaps, and to provide a research roadmap for a more extensive study planned as a current master thesis project.
@inproceedings{klag-poll-TEXT2KG2023,
author = {Gerhard Georg Klager and Axel Polleres},
abstract = {In this paper we report about preliminary results on running question answering benchmarks against the recently hyped conversational AI services such as ChatGPT: we focus on questions that are known to be possible to be answered by information in existing Knowledge graphs such as Wikidata. In a preliminary study we experiment, on the one hand, with questions from established KGQA benchmarks, and on the other hand, present a set of questions established in a student experiment, which should be particularly hard for Large Language Models (LLMs) to answer, mainly focusing on questions on recent events. In a second experiment, we assess how far GPT could be used for query generation in SPARQL. While our results are mostly negative for now, we hope to provide insights for further research in this direction, in terms of isolating and discussing the most obvious challenges and gaps, and to provide a research roadmap for a more extensive study planned as a current master thesis project.},
title = {Is {GPT} fit for {KGQA}? -- Preliminary results},
booktitle = {Proceedings of the International Workshop on
Knowledge Graph Generation from Text (Text2KG2023), co-located with Extended Semantic Web Conference 2023 (ESWC 2023)},
note = {to appear},
year = 2023,
day = 29,
month = may,
url = {http://polleres.net/publications/klag-poll-TEXT2KG2023.pdf}
}
Downloads: 0
{"_id":"5nxurKzZceCXbq3oB","bibbaseid":"klager-polleres-isgptfitforkgqapreliminaryresults-2023","author_short":["Klager, G. G.","Polleres, A."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","author":[{"firstnames":["Gerhard","Georg"],"propositions":[],"lastnames":["Klager"],"suffixes":[]},{"firstnames":["Axel"],"propositions":[],"lastnames":["Polleres"],"suffixes":[]}],"abstract":"In this paper we report about preliminary results on running question answering benchmarks against the recently hyped conversational AI services such as ChatGPT: we focus on questions that are known to be possible to be answered by information in existing Knowledge graphs such as Wikidata. In a preliminary study we experiment, on the one hand, with questions from established KGQA benchmarks, and on the other hand, present a set of questions established in a student experiment, which should be particularly hard for Large Language Models (LLMs) to answer, mainly focusing on questions on recent events. In a second experiment, we assess how far GPT could be used for query generation in SPARQL. While our results are mostly negative for now, we hope to provide insights for further research in this direction, in terms of isolating and discussing the most obvious challenges and gaps, and to provide a research roadmap for a more extensive study planned as a current master thesis project.","title":"Is GPT fit for KGQA? – Preliminary results","booktitle":"Proceedings of the International Workshop on Knowledge Graph Generation from Text (Text2KG2023), co-located with Extended Semantic Web Conference 2023 (ESWC 2023)","note":"to appear","year":"2023","day":"29","month":"May","url":"http://polleres.net/publications/klag-poll-TEXT2KG2023.pdf","bibtex":"@inproceedings{klag-poll-TEXT2KG2023,\nauthor = {Gerhard Georg Klager and Axel Polleres},\nabstract = {In this paper we report about preliminary results on running question answering benchmarks against the recently hyped conversational AI services such as ChatGPT: we focus on questions that are known to be possible to be answered by information in existing Knowledge graphs such as Wikidata. In a preliminary study we experiment, on the one hand, with questions from established KGQA benchmarks, and on the other hand, present a set of questions established in a student experiment, which should be particularly hard for Large Language Models (LLMs) to answer, mainly focusing on questions on recent events. In a second experiment, we assess how far GPT could be used for query generation in SPARQL. While our results are mostly negative for now, we hope to provide insights for further research in this direction, in terms of isolating and discussing the most obvious challenges and gaps, and to provide a research roadmap for a more extensive study planned as a current master thesis project.},\ntitle = {Is {GPT} fit for {KGQA}? -- Preliminary results},\nbooktitle = {Proceedings of the International Workshop on\nKnowledge Graph Generation from Text (Text2KG2023), co-located with Extended Semantic Web Conference 2023 (ESWC 2023)},\nnote = {to appear},\nyear = 2023,\nday = 29,\nmonth = may,\nurl = {http://polleres.net/publications/klag-poll-TEXT2KG2023.pdf}\n}\n\n\n","author_short":["Klager, G. G.","Polleres, A."],"key":"klag-poll-TEXT2KG2023","id":"klag-poll-TEXT2KG2023","bibbaseid":"klager-polleres-isgptfitforkgqapreliminaryresults-2023","role":"author","urls":{"Paper":"http://polleres.net/publications/klag-poll-TEXT2KG2023.pdf"},"metadata":{"authorlinks":{}},"html":""},"bibtype":"inproceedings","biburl":"www.polleres.net/mypublications.bib","dataSources":["gixxkiKt6rtWGoKSh","cBfwyqsLFQQMc4Fss"],"keywords":[],"search_terms":["gpt","fit","kgqa","preliminary","results","klager","polleres"],"title":"Is GPT fit for KGQA? – Preliminary results","year":2023}