@article{del_giudice_travelers_2021,
	title = {A {Traveler}’s {Guide} to the {Multiverse}: {Promises}, {Pitfalls}, and a {Framework} for the {Evaluation} of {Analytic} {Decisions}},
	volume = {4},
	issn = {2515-2459},
	shorttitle = {A {Traveler}’s {Guide} to the {Multiverse}},
	url = {https://doi.org/10.1177/2515245920954925},
	doi = {10.1177/2515245920954925},
	abstract = {Decisions made by researchers while analyzing data (e.g., how to measure variables, how to handle outliers) are sometimes arbitrary, without an objective justification for choosing one alternative over another. Multiverse-style methods (e.g., specification curve, vibration of effects) estimate an effect across an entire set of possible specifications to expose the impact of hidden degrees of freedom and/or obtain robust, less biased estimates of the effect of interest. However, if specifications are not truly arbitrary, multiverse-style analyses can produce misleading results, potentially hiding meaningful effects within a mass of poorly justified alternatives. So far, a key question has received scant attention: How does one decide whether alternatives are arbitrary? We offer a framework and conceptual tools for doing so. We discuss three kinds of a priori nonequivalence among alternatives—measurement nonequivalence, effect nonequivalence, and power/precision nonequivalence. The criteria we review lead to three decision scenarios: Type E decisions (principled equivalence), Type N decisions (principled nonequivalence), and Type U decisions (uncertainty). In uncertain scenarios, multiverse-style analysis should be conducted in a deliberately exploratory fashion. The framework is discussed with reference to published examples and illustrated with the help of a simulated data set. Our framework will help researchers reap the benefits of multiverse-style methods while avoiding their pitfalls.},
	language = {en},
	number = {1},
	urldate = {2021-08-13},
	journal = {Advances in Methods and Practices in Psychological Science},
	publisher = {SAGE Publications Inc},
	author = {Del Giudice, Marco and Gangestad, Steven W.},
	month = jan,
	year = {2021},
	keywords = {causal modeling, effects, equivalence, multiverse, open materials, psychometrics, robustness, specification curve, validity, vibration of effects},
	pages = {2515245920954925},
}

{"_id":"hiFGvt9nsYwvc7ZYC","bibbaseid":"delgiudice-gangestad-atravelersguidetothemultiversepromisespitfallsandaframeworkfortheevaluationofanalyticdecisions-2021","author_short":["Del Giudice, M.","Gangestad, S. W."],"bibdata":{"bibtype":"article","type":"article","title":"A Traveler’s Guide to the Multiverse: Promises, Pitfalls, and a Framework for the Evaluation of Analytic Decisions","volume":"4","issn":"2515-2459","shorttitle":"A Traveler’s Guide to the Multiverse","url":"https://doi.org/10.1177/2515245920954925","doi":"10.1177/2515245920954925","abstract":"Decisions made by researchers while analyzing data (e.g., how to measure variables, how to handle outliers) are sometimes arbitrary, without an objective justification for choosing one alternative over another. Multiverse-style methods (e.g., specification curve, vibration of effects) estimate an effect across an entire set of possible specifications to expose the impact of hidden degrees of freedom and/or obtain robust, less biased estimates of the effect of interest. However, if specifications are not truly arbitrary, multiverse-style analyses can produce misleading results, potentially hiding meaningful effects within a mass of poorly justified alternatives. So far, a key question has received scant attention: How does one decide whether alternatives are arbitrary? We offer a framework and conceptual tools for doing so. We discuss three kinds of a priori nonequivalence among alternatives—measurement nonequivalence, effect nonequivalence, and power/precision nonequivalence. The criteria we review lead to three decision scenarios: Type E decisions (principled equivalence), Type N decisions (principled nonequivalence), and Type U decisions (uncertainty). In uncertain scenarios, multiverse-style analysis should be conducted in a deliberately exploratory fashion. The framework is discussed with reference to published examples and illustrated with the help of a simulated data set. Our framework will help researchers reap the benefits of multiverse-style methods while avoiding their pitfalls.","language":"en","number":"1","urldate":"2021-08-13","journal":"Advances in Methods and Practices in Psychological Science","publisher":"SAGE Publications Inc","author":[{"propositions":[],"lastnames":["Del","Giudice"],"firstnames":["Marco"],"suffixes":[]},{"propositions":[],"lastnames":["Gangestad"],"firstnames":["Steven","W."],"suffixes":[]}],"month":"January","year":"2021","keywords":"causal modeling, effects, equivalence, multiverse, open materials, psychometrics, robustness, specification curve, validity, vibration of effects","pages":"2515245920954925","bibtex":"@article{del_giudice_travelers_2021,\n\ttitle = {A {Traveler}’s {Guide} to the {Multiverse}: {Promises}, {Pitfalls}, and a {Framework} for the {Evaluation} of {Analytic} {Decisions}},\n\tvolume = {4},\n\tissn = {2515-2459},\n\tshorttitle = {A {Traveler}’s {Guide} to the {Multiverse}},\n\turl = {https://doi.org/10.1177/2515245920954925},\n\tdoi = {10.1177/2515245920954925},\n\tabstract = {Decisions made by researchers while analyzing data (e.g., how to measure variables, how to handle outliers) are sometimes arbitrary, without an objective justification for choosing one alternative over another. Multiverse-style methods (e.g., specification curve, vibration of effects) estimate an effect across an entire set of possible specifications to expose the impact of hidden degrees of freedom and/or obtain robust, less biased estimates of the effect of interest. However, if specifications are not truly arbitrary, multiverse-style analyses can produce misleading results, potentially hiding meaningful effects within a mass of poorly justified alternatives. So far, a key question has received scant attention: How does one decide whether alternatives are arbitrary? We offer a framework and conceptual tools for doing so. We discuss three kinds of a priori nonequivalence among alternatives—measurement nonequivalence, effect nonequivalence, and power/precision nonequivalence. The criteria we review lead to three decision scenarios: Type E decisions (principled equivalence), Type N decisions (principled nonequivalence), and Type U decisions (uncertainty). In uncertain scenarios, multiverse-style analysis should be conducted in a deliberately exploratory fashion. The framework is discussed with reference to published examples and illustrated with the help of a simulated data set. Our framework will help researchers reap the benefits of multiverse-style methods while avoiding their pitfalls.},\n\tlanguage = {en},\n\tnumber = {1},\n\turldate = {2021-08-13},\n\tjournal = {Advances in Methods and Practices in Psychological Science},\n\tpublisher = {SAGE Publications Inc},\n\tauthor = {Del Giudice, Marco and Gangestad, Steven W.},\n\tmonth = jan,\n\tyear = {2021},\n\tkeywords = {causal modeling, effects, equivalence, multiverse, open materials, psychometrics, robustness, specification curve, validity, vibration of effects},\n\tpages = {2515245920954925},\n}\n\n\n\n\n\n\n\n\n\n\n\n","author_short":["Del Giudice, M.","Gangestad, S. W."],"key":"del_giudice_travelers_2021","id":"del_giudice_travelers_2021","bibbaseid":"delgiudice-gangestad-atravelersguidetothemultiversepromisespitfallsandaframeworkfortheevaluationofanalyticdecisions-2021","role":"author","urls":{"Paper":"https://doi.org/10.1177/2515245920954925"},"keyword":["causal modeling","effects","equivalence","multiverse","open materials","psychometrics","robustness","specification curve","validity","vibration of effects"],"metadata":{"authorlinks":{}},"html":""},"bibtype":"article","biburl":"https://bibbase.org/zotero/pab2163","dataSources":["LGwahddhyxfTkZijp","fB4GuzdCZcPR6LeBn"],"keywords":["causal modeling","effects","equivalence","multiverse","open materials","psychometrics","robustness","specification curve","validity","vibration of effects"],"search_terms":["traveler","guide","multiverse","promises","pitfalls","framework","evaluation","analytic","decisions","del giudice","gangestad"],"title":"A Traveler’s Guide to the Multiverse: Promises, Pitfalls, and a Framework for the Evaluation of Analytic Decisions","year":2021}