@article{eckhensha21,
	abstract = {Simulation optimization involves the optimization of some objective function that can only be estimated via stochastic simulation. Many important problems can be profitably viewed within this framework. While many solvers - implementations of simulation-optimization algorithms - exist or are in development, comparisons among solvers are not standardized and are often limited in scope. Such comparisons help advance solver development, clarify the relative performance of solvers and identify classes of problems that defy efficient solution, among many other uses. We develop performance measures and plots, and estimators thereof, that can be used to evaluate and compare solvers on a testbed of simulation-optimization problems. We explain the need for two-level simulation in this context and provide supporting theory to guide the use of common random numbers to achieve efficient comparisons. We also describe how to use bootstrapping to obtain error estimates for the estimators.},
	author = {David J. Eckman and Shane G. Henderson and Sara Shashaani},
	date-added = {2022-01-08 11:02:28 -0500},
	date-modified = {2023-04-13 08:11:22 -0400},
	journal = {{INFORMS} Journal on Computing},
	number = {2},
	pages = {350-367},
	title = {Diagnostic Tools for Evaluating and Comparing Simulation-Optimization Algorithms},
	url = {https://doi.org/10.1287/ijoc.2022.1261},
	url_paper = {pubs/comparingsimopt.pdf},
	volume = {35},
	year = {2023},
	bdsk-url-1 = {https://doi.org/10.1287/ijoc.2022.1261}}

{"_id":"RAakCo8jB89qjt2oP","bibbaseid":"eckman-henderson-shashaani-diagnostictoolsforevaluatingandcomparingsimulationoptimizationalgorithms-2023","author_short":["Eckman, D. J.","Henderson, S. G.","Shashaani, S."],"bibdata":{"bibtype":"article","type":"article","abstract":"Simulation optimization involves the optimization of some objective function that can only be estimated via stochastic simulation. Many important problems can be profitably viewed within this framework. While many solvers - implementations of simulation-optimization algorithms - exist or are in development, comparisons among solvers are not standardized and are often limited in scope. Such comparisons help advance solver development, clarify the relative performance of solvers and identify classes of problems that defy efficient solution, among many other uses. We develop performance measures and plots, and estimators thereof, that can be used to evaluate and compare solvers on a testbed of simulation-optimization problems. We explain the need for two-level simulation in this context and provide supporting theory to guide the use of common random numbers to achieve efficient comparisons. We also describe how to use bootstrapping to obtain error estimates for the estimators.","author":[{"firstnames":["David","J."],"propositions":[],"lastnames":["Eckman"],"suffixes":[]},{"firstnames":["Shane","G."],"propositions":[],"lastnames":["Henderson"],"suffixes":[]},{"firstnames":["Sara"],"propositions":[],"lastnames":["Shashaani"],"suffixes":[]}],"date-added":"2022-01-08 11:02:28 -0500","date-modified":"2023-04-13 08:11:22 -0400","journal":"INFORMS Journal on Computing","number":"2","pages":"350-367","title":"Diagnostic Tools for Evaluating and Comparing Simulation-Optimization Algorithms","url":"https://doi.org/10.1287/ijoc.2022.1261","url_paper":"pubs/comparingsimopt.pdf","volume":"35","year":"2023","bdsk-url-1":"https://doi.org/10.1287/ijoc.2022.1261","bibtex":"@article{eckhensha21,\n\tabstract = {Simulation optimization involves the optimization of some objective function that can only be estimated via stochastic simulation. Many important problems can be profitably viewed within this framework. While many solvers - implementations of simulation-optimization algorithms - exist or are in development, comparisons among solvers are not standardized and are often limited in scope. Such comparisons help advance solver development, clarify the relative performance of solvers and identify classes of problems that defy efficient solution, among many other uses. We develop performance measures and plots, and estimators thereof, that can be used to evaluate and compare solvers on a testbed of simulation-optimization problems. We explain the need for two-level simulation in this context and provide supporting theory to guide the use of common random numbers to achieve efficient comparisons. We also describe how to use bootstrapping to obtain error estimates for the estimators.},\n\tauthor = {David J. Eckman and Shane G. Henderson and Sara Shashaani},\n\tdate-added = {2022-01-08 11:02:28 -0500},\n\tdate-modified = {2023-04-13 08:11:22 -0400},\n\tjournal = {{INFORMS} Journal on Computing},\n\tnumber = {2},\n\tpages = {350-367},\n\ttitle = {Diagnostic Tools for Evaluating and Comparing Simulation-Optimization Algorithms},\n\turl = {https://doi.org/10.1287/ijoc.2022.1261},\n\turl_paper = {pubs/comparingsimopt.pdf},\n\tvolume = {35},\n\tyear = {2023},\n\tbdsk-url-1 = {https://doi.org/10.1287/ijoc.2022.1261}}\n\n","author_short":["Eckman, D. J.","Henderson, S. G.","Shashaani, S."],"key":"eckhensha21","id":"eckhensha21","bibbaseid":"eckman-henderson-shashaani-diagnostictoolsforevaluatingandcomparingsimulationoptimizationalgorithms-2023","role":"author","urls":{"Paper":"https://doi.org/10.1287/ijoc.2022.1261"," paper":"https://people.orie.cornell.edu/shane/pubs/comparingsimopt.pdf"},"metadata":{"authorlinks":{}},"downloads":14},"bibtype":"article","biburl":"https://people.orie.cornell.edu/shane/ShanePubs.bib","dataSources":["ZCuKDjctePZJeeaBw"],"keywords":[],"search_terms":["diagnostic","tools","evaluating","comparing","simulation","optimization","algorithms","eckman","henderson","shashaani"],"title":"Diagnostic Tools for Evaluating and Comparing Simulation-Optimization Algorithms","year":2023,"downloads":14}