@article{RN397,
   author = {Reps, J. M. and Kim, C. and Williams, R. D. and Markus, A. F. and Yang, C. and Duarte-Salles, T. and Falconer, T. and Jonnagaddala, J. and Williams, A. and Fernández-Bertolín, S. and DuVall, S. L. and Kostka, K. and Rao, G. and Shoaibi, A. and Ostropolets, A. and Spotnitz, M. E. and Zhang, L. and Casajust, P. and Steyerberg, E. W. and Nyberg, F. and Kaas-Hansen, B. S. and Choi, Y. H. and Morales, D. and Liaw, S. T. and Abrahão, M. T. F. and Areia, C. and Matheny, M. E. and Lynch, K. E. and Aragón, M. and Park, R. W. and Hripcsak, G. and Reich, C. G. and Suchard, M. A. and You, S. C. and Ryan, P. B. and Prieto-Alhambra, D. and Rijnbeek, P. R.},
   title = {Implementation of the COVID-19 Vulnerability Index Across an International Network of Health Care Data Sets: Collaborative External Validation Study},
   journal = {JMIR Med Inform},
   volume = {9},
   number = {4},
   pages = {e21547},
   note = {2291-9694 Reps, Jenna M Orcid: 0000-0002-2970-0778 Kim, Chungsoo Orcid: 0000-0003-1802-1777 Williams, Ross D Orcid: 0000-0001-7723-417x Markus, Aniek F Orcid: 0000-0001-5779-4794 Yang, Cynthia Orcid: 0000-0001-6769-3153 Duarte-Salles, Talita Orcid: 0000-0002-8274-0357 Falconer, Thomas Orcid: 0000-0001-6904-1394 Jonnagaddala, Jitendra Orcid: 0000-0002-9912-2344 Williams, Andrew Orcid: 0000-0002-0692-412x Fernández-Bertolín, Sergio Orcid: 0000-0002-0029-6564 DuVall, Scott L Orcid: 0000-0002-4898-3865 Kostka, Kristin Orcid: 0000-0003-2595-8736 Rao, Gowtham Orcid: 0000-0002-4949-7236 Shoaibi, Azza Orcid: 0000-0002-6976-2594 Ostropolets, Anna Orcid: 0000-0002-0847-6682 Spotnitz, Matthew E Orcid: 0000-0003-2869-0237 Zhang, Lin Orcid: 0000-0002-2064-8440 Casajust, Paula Orcid: 0000-0003-2733-5436 Steyerberg, Ewout W Orcid: 0000-0002-7787-0122 Nyberg, Fredrik Orcid: 0000-0003-0892-5668 Kaas-Hansen, Benjamin Skov Orcid: 0000-0003-1023-0371 Choi, Young Hwa Orcid: 0000-0001-5254-3101 Morales, Daniel Orcid: 0000-0002-0063-8069 Liaw, Siaw-Teng Orcid: 0000-0001-5989-3614 Abrahão, Maria Tereza Fernandes Orcid: 0000-0003-2701-670x Areia, Carlos Orcid: 0000-0002-4668-7069 Matheny, Michael E Orcid: 0000-0003-3217-4147 Lynch, Kristine E Orcid: 0000-0002-2221-2421 Aragón, María Orcid: 0000-0002-5990-6953 Park, Rae Woong Orcid: 0000-0003-4989-3287 Hripcsak, George Orcid: 0000-0003-2664-7614 Reich, Christian G Orcid: 0000-0002-3641-055x Suchard, Marc A Orcid: 0000-0001-9818-479x You, Seng Chan Orcid: 0000-0002-5052-6399 Ryan, Patrick B Orcid: 0000-0002-9727-2138 Prieto-Alhambra, Daniel Orcid: 0000-0002-3950-6346 Rijnbeek, Peter R Orcid: 0000-0003-0621-1979 R01 LM006910/LM/NLM NIH HHS/United States T15 LM007079/LM/NLM NIH HHS/United States Journal Article Canada 2021/03/05 JMIR Med Inform. 2021 Apr 5;9(4):e21547. doi: 10.2196/21547.},
   abstract = {BACKGROUND: SARS-CoV-2 is straining health care systems globally. The burden on hospitals during the pandemic could be reduced by implementing prediction models that can discriminate patients who require hospitalization from those who do not. The COVID-19 vulnerability (C-19) index, a model that predicts which patients will be admitted to hospital for treatment of pneumonia or pneumonia proxies, has been developed and proposed as a valuable tool for decision-making during the pandemic. However, the model is at high risk of bias according to the prediction model risk of bias assessment criteria, and it has not been externally validated. OBJECTIVE: The aim of this study was to externally validate the C-19 index across a range of health care settings to determine how well it broadly predicts hospitalization due to pneumonia in COVID-19 cases. METHODS: We followed the Observational Health Data Sciences and Informatics (OHDSI) framework for external validation to assess the reliability of the C-19 index. We evaluated the model on two different target populations, 41,381 patients who presented with SARS-CoV-2 at an outpatient or emergency department visit and 9,429,285 patients who presented with influenza or related symptoms during an outpatient or emergency department visit, to predict their risk of hospitalization with pneumonia during the following 0-30 days. In total, we validated the model across a network of 14 databases spanning the United States, Europe, Australia, and Asia. RESULTS: The internal validation performance of the C-19 index had a C statistic of 0.73, and the calibration was not reported by the authors. When we externally validated it by transporting it to SARS-CoV-2 data, the model obtained C statistics of 0.36, 0.53 (0.473-0.584) and 0.56 (0.488-0.636) on Spanish, US, and South Korean data sets, respectively. The calibration was poor, with the model underestimating risk. When validated on 12 data sets containing influenza patients across the OHDSI network, the C statistics ranged between 0.40 and 0.68. CONCLUSIONS: Our results show that the discriminative performance of the C-19 index model is low for influenza cohorts and even worse among patients with COVID-19 in the United States, Spain, and South Korea. These results suggest that C-19 should not be used to aid decision-making during the COVID-19 pandemic. Our findings highlight the importance of performing external validation across a range of settings, especially when a prediction model is being extrapolated to a different population. In the field of prediction, extensive validation is required to create appropriate trust in a model.},
   keywords = {C-19
Covid-19
bias
datasets
decision-making
external validation
hospitalization
modeling
observation
prediction
prognostic model
risk
transportability ,
ISSN = 2291-9694 (Print) ,
DOI = 10.2196/21547},
   year = {2021},
   type = {Journal Article}
}

{"_id":"uggtuQqnfn6hvTC3q","bibbaseid":"reps-kim-williams-markus-yang-duartesalles-falconer-jonnagaddala-etal-implementationofthecovid19vulnerabilityindexacrossaninternationalnetworkofhealthcaredatasetscollaborativeexternalvalidationstudy-2021","author_short":["Reps, J. M.","Kim, C.","Williams, R. D.","Markus, A. F.","Yang, C.","Duarte-Salles, T.","Falconer, T.","Jonnagaddala, J.","Williams, A.","Fernández-Bertolín, S.","DuVall, S. L.","Kostka, K.","Rao, G.","Shoaibi, A.","Ostropolets, A.","Spotnitz, M. E.","Zhang, L.","Casajust, P.","Steyerberg, E. W.","Nyberg, F.","Kaas-Hansen, B. S.","Choi, Y. H.","Morales, D.","Liaw, S. T.","Abrahão, M. T. F.","Areia, C.","Matheny, M. E.","Lynch, K. E.","Aragón, M.","Park, R. W.","Hripcsak, G.","Reich, C. G.","Suchard, M. A.","You, S. C.","Ryan, P. B.","Prieto-Alhambra, D.","Rijnbeek, P. R."],"bibdata":{"bibtype":"article","type":"Journal Article","author":[{"propositions":[],"lastnames":["Reps"],"firstnames":["J.","M."],"suffixes":[]},{"propositions":[],"lastnames":["Kim"],"firstnames":["C."],"suffixes":[]},{"propositions":[],"lastnames":["Williams"],"firstnames":["R.","D."],"suffixes":[]},{"propositions":[],"lastnames":["Markus"],"firstnames":["A.","F."],"suffixes":[]},{"propositions":[],"lastnames":["Yang"],"firstnames":["C."],"suffixes":[]},{"propositions":[],"lastnames":["Duarte-Salles"],"firstnames":["T."],"suffixes":[]},{"propositions":[],"lastnames":["Falconer"],"firstnames":["T."],"suffixes":[]},{"propositions":[],"lastnames":["Jonnagaddala"],"firstnames":["J."],"suffixes":[]},{"propositions":[],"lastnames":["Williams"],"firstnames":["A."],"suffixes":[]},{"propositions":[],"lastnames":["Fernández-Bertolín"],"firstnames":["S."],"suffixes":[]},{"propositions":[],"lastnames":["DuVall"],"firstnames":["S.","L."],"suffixes":[]},{"propositions":[],"lastnames":["Kostka"],"firstnames":["K."],"suffixes":[]},{"propositions":[],"lastnames":["Rao"],"firstnames":["G."],"suffixes":[]},{"propositions":[],"lastnames":["Shoaibi"],"firstnames":["A."],"suffixes":[]},{"propositions":[],"lastnames":["Ostropolets"],"firstnames":["A."],"suffixes":[]},{"propositions":[],"lastnames":["Spotnitz"],"firstnames":["M.","E."],"suffixes":[]},{"propositions":[],"lastnames":["Zhang"],"firstnames":["L."],"suffixes":[]},{"propositions":[],"lastnames":["Casajust"],"firstnames":["P."],"suffixes":[]},{"propositions":[],"lastnames":["Steyerberg"],"firstnames":["E.","W."],"suffixes":[]},{"propositions":[],"lastnames":["Nyberg"],"firstnames":["F."],"suffixes":[]},{"propositions":[],"lastnames":["Kaas-Hansen"],"firstnames":["B.","S."],"suffixes":[]},{"propositions":[],"lastnames":["Choi"],"firstnames":["Y.","H."],"suffixes":[]},{"propositions":[],"lastnames":["Morales"],"firstnames":["D."],"suffixes":[]},{"propositions":[],"lastnames":["Liaw"],"firstnames":["S.","T."],"suffixes":[]},{"propositions":[],"lastnames":["Abrahão"],"firstnames":["M.","T.","F."],"suffixes":[]},{"propositions":[],"lastnames":["Areia"],"firstnames":["C."],"suffixes":[]},{"propositions":[],"lastnames":["Matheny"],"firstnames":["M.","E."],"suffixes":[]},{"propositions":[],"lastnames":["Lynch"],"firstnames":["K.","E."],"suffixes":[]},{"propositions":[],"lastnames":["Aragón"],"firstnames":["M."],"suffixes":[]},{"propositions":[],"lastnames":["Park"],"firstnames":["R.","W."],"suffixes":[]},{"propositions":[],"lastnames":["Hripcsak"],"firstnames":["G."],"suffixes":[]},{"propositions":[],"lastnames":["Reich"],"firstnames":["C.","G."],"suffixes":[]},{"propositions":[],"lastnames":["Suchard"],"firstnames":["M.","A."],"suffixes":[]},{"propositions":[],"lastnames":["You"],"firstnames":["S.","C."],"suffixes":[]},{"propositions":[],"lastnames":["Ryan"],"firstnames":["P.","B."],"suffixes":[]},{"propositions":[],"lastnames":["Prieto-Alhambra"],"firstnames":["D."],"suffixes":[]},{"propositions":[],"lastnames":["Rijnbeek"],"firstnames":["P.","R."],"suffixes":[]}],"title":"Implementation of the COVID-19 Vulnerability Index Across an International Network of Health Care Data Sets: Collaborative External Validation Study","journal":"JMIR Med Inform","volume":"9","number":"4","pages":"e21547","note":"2291-9694 Reps, Jenna M Orcid: 0000-0002-2970-0778 Kim, Chungsoo Orcid: 0000-0003-1802-1777 Williams, Ross D Orcid: 0000-0001-7723-417x Markus, Aniek F Orcid: 0000-0001-5779-4794 Yang, Cynthia Orcid: 0000-0001-6769-3153 Duarte-Salles, Talita Orcid: 0000-0002-8274-0357 Falconer, Thomas Orcid: 0000-0001-6904-1394 Jonnagaddala, Jitendra Orcid: 0000-0002-9912-2344 Williams, Andrew Orcid: 0000-0002-0692-412x Fernández-Bertolín, Sergio Orcid: 0000-0002-0029-6564 DuVall, Scott L Orcid: 0000-0002-4898-3865 Kostka, Kristin Orcid: 0000-0003-2595-8736 Rao, Gowtham Orcid: 0000-0002-4949-7236 Shoaibi, Azza Orcid: 0000-0002-6976-2594 Ostropolets, Anna Orcid: 0000-0002-0847-6682 Spotnitz, Matthew E Orcid: 0000-0003-2869-0237 Zhang, Lin Orcid: 0000-0002-2064-8440 Casajust, Paula Orcid: 0000-0003-2733-5436 Steyerberg, Ewout W Orcid: 0000-0002-7787-0122 Nyberg, Fredrik Orcid: 0000-0003-0892-5668 Kaas-Hansen, Benjamin Skov Orcid: 0000-0003-1023-0371 Choi, Young Hwa Orcid: 0000-0001-5254-3101 Morales, Daniel Orcid: 0000-0002-0063-8069 Liaw, Siaw-Teng Orcid: 0000-0001-5989-3614 Abrahão, Maria Tereza Fernandes Orcid: 0000-0003-2701-670x Areia, Carlos Orcid: 0000-0002-4668-7069 Matheny, Michael E Orcid: 0000-0003-3217-4147 Lynch, Kristine E Orcid: 0000-0002-2221-2421 Aragón, María Orcid: 0000-0002-5990-6953 Park, Rae Woong Orcid: 0000-0003-4989-3287 Hripcsak, George Orcid: 0000-0003-2664-7614 Reich, Christian G Orcid: 0000-0002-3641-055x Suchard, Marc A Orcid: 0000-0001-9818-479x You, Seng Chan Orcid: 0000-0002-5052-6399 Ryan, Patrick B Orcid: 0000-0002-9727-2138 Prieto-Alhambra, Daniel Orcid: 0000-0002-3950-6346 Rijnbeek, Peter R Orcid: 0000-0003-0621-1979 R01 LM006910/LM/NLM NIH HHS/United States T15 LM007079/LM/NLM NIH HHS/United States Journal Article Canada 2021/03/05 JMIR Med Inform. 2021 Apr 5;9(4):e21547. doi: 10.2196/21547.","abstract":"BACKGROUND: SARS-CoV-2 is straining health care systems globally. The burden on hospitals during the pandemic could be reduced by implementing prediction models that can discriminate patients who require hospitalization from those who do not. The COVID-19 vulnerability (C-19) index, a model that predicts which patients will be admitted to hospital for treatment of pneumonia or pneumonia proxies, has been developed and proposed as a valuable tool for decision-making during the pandemic. However, the model is at high risk of bias according to the prediction model risk of bias assessment criteria, and it has not been externally validated. OBJECTIVE: The aim of this study was to externally validate the C-19 index across a range of health care settings to determine how well it broadly predicts hospitalization due to pneumonia in COVID-19 cases. METHODS: We followed the Observational Health Data Sciences and Informatics (OHDSI) framework for external validation to assess the reliability of the C-19 index. We evaluated the model on two different target populations, 41,381 patients who presented with SARS-CoV-2 at an outpatient or emergency department visit and 9,429,285 patients who presented with influenza or related symptoms during an outpatient or emergency department visit, to predict their risk of hospitalization with pneumonia during the following 0-30 days. In total, we validated the model across a network of 14 databases spanning the United States, Europe, Australia, and Asia. RESULTS: The internal validation performance of the C-19 index had a C statistic of 0.73, and the calibration was not reported by the authors. When we externally validated it by transporting it to SARS-CoV-2 data, the model obtained C statistics of 0.36, 0.53 (0.473-0.584) and 0.56 (0.488-0.636) on Spanish, US, and South Korean data sets, respectively. The calibration was poor, with the model underestimating risk. When validated on 12 data sets containing influenza patients across the OHDSI network, the C statistics ranged between 0.40 and 0.68. CONCLUSIONS: Our results show that the discriminative performance of the C-19 index model is low for influenza cohorts and even worse among patients with COVID-19 in the United States, Spain, and South Korea. These results suggest that C-19 should not be used to aid decision-making during the COVID-19 pandemic. Our findings highlight the importance of performing external validation across a range of settings, especially when a prediction model is being extrapolated to a different population. In the field of prediction, extensive validation is required to create appropriate trust in a model.","keywords":"C-19 Covid-19 bias datasets decision-making external validation hospitalization modeling observation prediction prognostic model risk transportability , ISSN = 2291-9694 (Print) , DOI = 10.2196/21547","year":"2021","bibtex":"@article{RN397,\r\n author = {Reps, J. M. and Kim, C. and Williams, R. D. and Markus, A. F. and Yang, C. and Duarte-Salles, T. and Falconer, T. and Jonnagaddala, J. and Williams, A. and Fernández-Bertolín, S. and DuVall, S. L. and Kostka, K. and Rao, G. and Shoaibi, A. and Ostropolets, A. and Spotnitz, M. E. and Zhang, L. and Casajust, P. and Steyerberg, E. W. and Nyberg, F. and Kaas-Hansen, B. S. and Choi, Y. H. and Morales, D. and Liaw, S. T. and Abrahão, M. T. F. and Areia, C. and Matheny, M. E. and Lynch, K. E. and Aragón, M. and Park, R. W. and Hripcsak, G. and Reich, C. G. and Suchard, M. A. and You, S. C. and Ryan, P. B. and Prieto-Alhambra, D. and Rijnbeek, P. R.},\r\n title = {Implementation of the COVID-19 Vulnerability Index Across an International Network of Health Care Data Sets: Collaborative External Validation Study},\r\n journal = {JMIR Med Inform},\r\n volume = {9},\r\n number = {4},\r\n pages = {e21547},\r\n note = {2291-9694 Reps, Jenna M Orcid: 0000-0002-2970-0778 Kim, Chungsoo Orcid: 0000-0003-1802-1777 Williams, Ross D Orcid: 0000-0001-7723-417x Markus, Aniek F Orcid: 0000-0001-5779-4794 Yang, Cynthia Orcid: 0000-0001-6769-3153 Duarte-Salles, Talita Orcid: 0000-0002-8274-0357 Falconer, Thomas Orcid: 0000-0001-6904-1394 Jonnagaddala, Jitendra Orcid: 0000-0002-9912-2344 Williams, Andrew Orcid: 0000-0002-0692-412x Fernández-Bertolín, Sergio Orcid: 0000-0002-0029-6564 DuVall, Scott L Orcid: 0000-0002-4898-3865 Kostka, Kristin Orcid: 0000-0003-2595-8736 Rao, Gowtham Orcid: 0000-0002-4949-7236 Shoaibi, Azza Orcid: 0000-0002-6976-2594 Ostropolets, Anna Orcid: 0000-0002-0847-6682 Spotnitz, Matthew E Orcid: 0000-0003-2869-0237 Zhang, Lin Orcid: 0000-0002-2064-8440 Casajust, Paula Orcid: 0000-0003-2733-5436 Steyerberg, Ewout W Orcid: 0000-0002-7787-0122 Nyberg, Fredrik Orcid: 0000-0003-0892-5668 Kaas-Hansen, Benjamin Skov Orcid: 0000-0003-1023-0371 Choi, Young Hwa Orcid: 0000-0001-5254-3101 Morales, Daniel Orcid: 0000-0002-0063-8069 Liaw, Siaw-Teng Orcid: 0000-0001-5989-3614 Abrahão, Maria Tereza Fernandes Orcid: 0000-0003-2701-670x Areia, Carlos Orcid: 0000-0002-4668-7069 Matheny, Michael E Orcid: 0000-0003-3217-4147 Lynch, Kristine E Orcid: 0000-0002-2221-2421 Aragón, María Orcid: 0000-0002-5990-6953 Park, Rae Woong Orcid: 0000-0003-4989-3287 Hripcsak, George Orcid: 0000-0003-2664-7614 Reich, Christian G Orcid: 0000-0002-3641-055x Suchard, Marc A Orcid: 0000-0001-9818-479x You, Seng Chan Orcid: 0000-0002-5052-6399 Ryan, Patrick B Orcid: 0000-0002-9727-2138 Prieto-Alhambra, Daniel Orcid: 0000-0002-3950-6346 Rijnbeek, Peter R Orcid: 0000-0003-0621-1979 R01 LM006910/LM/NLM NIH HHS/United States T15 LM007079/LM/NLM NIH HHS/United States Journal Article Canada 2021/03/05 JMIR Med Inform. 2021 Apr 5;9(4):e21547. doi: 10.2196/21547.},\r\n abstract = {BACKGROUND: SARS-CoV-2 is straining health care systems globally. The burden on hospitals during the pandemic could be reduced by implementing prediction models that can discriminate patients who require hospitalization from those who do not. The COVID-19 vulnerability (C-19) index, a model that predicts which patients will be admitted to hospital for treatment of pneumonia or pneumonia proxies, has been developed and proposed as a valuable tool for decision-making during the pandemic. However, the model is at high risk of bias according to the prediction model risk of bias assessment criteria, and it has not been externally validated. OBJECTIVE: The aim of this study was to externally validate the C-19 index across a range of health care settings to determine how well it broadly predicts hospitalization due to pneumonia in COVID-19 cases. METHODS: We followed the Observational Health Data Sciences and Informatics (OHDSI) framework for external validation to assess the reliability of the C-19 index. We evaluated the model on two different target populations, 41,381 patients who presented with SARS-CoV-2 at an outpatient or emergency department visit and 9,429,285 patients who presented with influenza or related symptoms during an outpatient or emergency department visit, to predict their risk of hospitalization with pneumonia during the following 0-30 days. In total, we validated the model across a network of 14 databases spanning the United States, Europe, Australia, and Asia. RESULTS: The internal validation performance of the C-19 index had a C statistic of 0.73, and the calibration was not reported by the authors. When we externally validated it by transporting it to SARS-CoV-2 data, the model obtained C statistics of 0.36, 0.53 (0.473-0.584) and 0.56 (0.488-0.636) on Spanish, US, and South Korean data sets, respectively. The calibration was poor, with the model underestimating risk. When validated on 12 data sets containing influenza patients across the OHDSI network, the C statistics ranged between 0.40 and 0.68. CONCLUSIONS: Our results show that the discriminative performance of the C-19 index model is low for influenza cohorts and even worse among patients with COVID-19 in the United States, Spain, and South Korea. These results suggest that C-19 should not be used to aid decision-making during the COVID-19 pandemic. Our findings highlight the importance of performing external validation across a range of settings, especially when a prediction model is being extrapolated to a different population. In the field of prediction, extensive validation is required to create appropriate trust in a model.},\r\n keywords = {C-19\r\nCovid-19\r\nbias\r\ndatasets\r\ndecision-making\r\nexternal validation\r\nhospitalization\r\nmodeling\r\nobservation\r\nprediction\r\nprognostic model\r\nrisk\r\ntransportability ,\r\nISSN = 2291-9694 (Print) ,\r\nDOI = 10.2196/21547},\r\n year = {2021},\r\n type = {Journal Article}\r\n}\r\n\r\n","author_short":["Reps, J. M.","Kim, C.","Williams, R. D.","Markus, A. F.","Yang, C.","Duarte-Salles, T.","Falconer, T.","Jonnagaddala, J.","Williams, A.","Fernández-Bertolín, S.","DuVall, S. L.","Kostka, K.","Rao, G.","Shoaibi, A.","Ostropolets, A.","Spotnitz, M. E.","Zhang, L.","Casajust, P.","Steyerberg, E. W.","Nyberg, F.","Kaas-Hansen, B. S.","Choi, Y. H.","Morales, D.","Liaw, S. T.","Abrahão, M. T. F.","Areia, C.","Matheny, M. E.","Lynch, K. E.","Aragón, M.","Park, R. W.","Hripcsak, G.","Reich, C. G.","Suchard, M. A.","You, S. C.","Ryan, P. B.","Prieto-Alhambra, D.","Rijnbeek, P. R."],"key":"RN397","id":"RN397","bibbaseid":"reps-kim-williams-markus-yang-duartesalles-falconer-jonnagaddala-etal-implementationofthecovid19vulnerabilityindexacrossaninternationalnetworkofhealthcaredatasetscollaborativeexternalvalidationstudy-2021","role":"author","urls":{},"keyword":["C-19 Covid-19 bias datasets decision-making external validation hospitalization modeling observation prediction prognostic model risk transportability","ISSN = 2291-9694 (Print)","DOI = 10.2196/21547"],"metadata":{"authorlinks":{}},"html":""},"bibtype":"article","biburl":"https://bibbase.org/network/files/qwipow2DJsAxqNp83","dataSources":["gYHzosaj6P7Z4gkGW","PJSqX8FMpCPRdw3CQ","qLDnCcKM9omXNrRY8","TB9sD3avjy7bNxfgS","vDmJPYmHEKAT3tMNL","PcZMFHooKwpboi2he","jZMXEeHTfokmisJDv","LKj7QkGgRgaPTmbcp"],"keywords":["c-19 covid-19 bias datasets decision-making external validation hospitalization modeling observation prediction prognostic model risk transportability","issn = 2291-9694 (print)","doi = 10.2196/21547"],"search_terms":["implementation","covid","vulnerability","index","international","network","health","care","data","sets","collaborative","external","validation","study","reps","kim","williams","markus","yang","duarte-salles","falconer","jonnagaddala","williams","fernández-bertolín","duvall","kostka","rao","shoaibi","ostropolets","spotnitz","zhang","casajust","steyerberg","nyberg","kaas-hansen","choi","morales","liaw","abrahão","areia","matheny","lynch","aragón","park","hripcsak","reich","suchard","you","ryan","prieto-alhambra","rijnbeek"],"title":"Implementation of the COVID-19 Vulnerability Index Across an International Network of Health Care Data Sets: Collaborative External Validation Study","year":2021}