var bibbase_data = {"data":"<img src=\"//bibbase.org/img/ajax-loader.gif\" id=\"spinner\"\n  style=\"display: none;\" alt=\"Loading..\" />\n\n<div id=\"bibbase\">\n\n  <script type=\"text/javascript\">\n    var bibbase = {\n      params: {\"bib\":\"https://api.zotero.org/users/6655/collections/7SR56WJZ/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&format=bibtex&limit=100\",\"jsonp\":\"1\",\"owner\":\"none\",\"host\":\"bibbase.org\"},\n      data: [{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Measuring fairness in ranked results: an analytical and empirical comparison\",\"url\":\"https://md.ekstrandom.net/pubs/fair-ranking\",\"doi\":\"10.1145/3477495.3532018\",\"abstract\":\"Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking. We aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.\",\"booktitle\":\"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval\",\"publisher\":\"ACM\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2022\",\"pages\":\"726–736\",\"bibtex\":\"@inproceedings{raj_measuring_2022,\\n\\ttitle = {Measuring fairness in ranked results: an analytical and empirical comparison},\\n\\turl = {https://md.ekstrandom.net/pubs/fair-ranking},\\n\\tdoi = {10.1145/3477495.3532018},\\n\\tabstract = {Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking.\\n\\nWe aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.},\\n\\tbooktitle = {Proceedings of the 45th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\\n\\tpublisher = {ACM},\\n\\tauthor = {Raj, Amifa and Ekstrand, Michael D},\\n\\tmonth = jul,\\n\\tyear = {2022},\\n\\tpages = {726--736},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Ekstrand, M. D\"],\"key\":\"raj_measuring_2022\",\"id\":\"raj_measuring_2022\",\"bibbaseid\":\"raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/fair-ranking\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":4,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"LNCS\",\"title\":\"Towards optimizing ranking in grid-layout for provider-side fairness\",\"volume\":\"14612\",\"copyright\":\"All rights reserved\",\"url\":\"https://md.ekstrandom.net/pubs/ecir-fair-grids\",\"doi\":\"10.1007/978-3-031-56069-9_7\",\"abstract\":\"Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.\",\"language\":\"en\",\"urldate\":\"2024-01-04\",\"booktitle\":\"Advances in Information Retrieval\",\"publisher\":\"Springer\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"March\",\"year\":\"2024\",\"pages\":\"90–105\",\"bibtex\":\"@inproceedings{raj_towards_2024,\\n\\tseries = {{LNCS}},\\n\\ttitle = {Towards optimizing ranking in grid-layout for provider-side fairness},\\n\\tvolume = {14612},\\n\\tcopyright = {All rights reserved},\\n\\turl = {https://md.ekstrandom.net/pubs/ecir-fair-grids},\\n\\tdoi = {10.1007/978-3-031-56069-9_7},\\n\\tabstract = {Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.},\\n\\tlanguage = {en},\\n\\turldate = {2024-01-04},\\n\\tbooktitle = {Advances in {Information} {Retrieval}},\\n\\tpublisher = {Springer},\\n\\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\\n\\tmonth = mar,\\n\\tyear = {2024},\\n\\tpages = {90--105},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Ekstrand, M. D.\"],\"key\":\"raj_towards_2024\",\"id\":\"raj_towards_2024\",\"bibbaseid\":\"raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Comparing fair ranking metrics\",\"url\":\"http://arxiv.org/abs/2009.01311\",\"abstract\":\"Ranking is a fundamental aspect of recommender systems. However, ranked outputs can be susceptible to various biases; some of these may cause disadvantages to members of protected groups. Several metrics have been proposed to quantify the (un)fairness of rankings, but there has not been to date any direct comparison of these metrics. This complicates deciding what fairness metrics are applicable for specific scenarios, and assessing the extent to which metrics agree or disagree. In this paper, we describe several fair ranking metrics in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data set. Our work provides a direct comparative analysis identifying similarities and differences of fair ranking metrics selected for our work.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wood\"],\"firstnames\":[\"Connor\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Montoly\"],\"firstnames\":[\"Ananda\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2020\",\"bibtex\":\"@inproceedings{raj_comparing_2020,\\n\\ttitle = {Comparing fair ranking metrics},\\n\\turl = {http://arxiv.org/abs/2009.01311},\\n\\tabstract = {Ranking is a fundamental aspect of recommender systems. However, ranked\\noutputs can be susceptible to various biases; some of these may cause\\ndisadvantages to members of protected groups. Several metrics have been\\nproposed to quantify the (un)fairness of rankings, but there has not been\\nto date any direct comparison of these metrics. This complicates deciding\\nwhat fairness metrics are applicable for specific scenarios, and assessing\\nthe extent to which metrics agree or disagree. In this paper, we describe\\nseveral fair ranking metrics in a common notation, enabling direct\\ncomparison of their approaches and assumptions, and empirically compare\\nthem on the same experimental setup and data set. Our work provides a\\ndirect comparative analysis identifying similarities and differences of\\nfair ranking metrics selected for our work.},\\n\\tauthor = {Raj, Amifa and Wood, Connor and Montoly, Ananda and Ekstrand, Michael D},\\n\\tmonth = sep,\\n\\tyear = {2020},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Wood, C.\",\"Montoly, A.\",\"Ekstrand, M. D\"],\"key\":\"raj_comparing_2020\",\"id\":\"raj_comparing_2020\",\"bibbaseid\":\"raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2009.01311\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":0,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"title\":\"Estimation of fair ranking metrics with incomplete judgments\",\"url\":\"https://arxiv.org/abs/2108.05152\",\"doi\":\"10.1145/3442381.3450080\",\"abstract\":\"There is increasing attention to evaluating the fairness of search system ranking decisions. These metrics often consider the membership of items to particular groups, often identified using protected attributes such as gender or ethnicity. To date, these metrics typically assume the availability and completeness of protected attribute labels of items. However, the protected attributes of individuals are rarely present, limiting the application of fair ranking metrics in large scale systems. In order to address this problem, we propose a sampling strategy and estimation technique for four fair ranking metrics. We formulate a robust and unbiased estimator which can operate even with very limited number of labeled items. We evaluate our approach using both simulated and real world data. Our experimental results demonstrate that our method can estimate this family of fair ranking metrics and provides a robust, reliable alternative to exhaustive or random data annotation.\",\"urldate\":\"2021-04-27\",\"booktitle\":\"WWW '21\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Kırnap\"],\"firstnames\":[\"Ömer\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Diaz\"],\"firstnames\":[\"Fernando\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Biega\"],\"firstnames\":[\"Asia\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Carterette\"],\"firstnames\":[\"Ben\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Yilmaz\"],\"firstnames\":[\"Emine\"],\"suffixes\":[]}],\"month\":\"April\",\"year\":\"2021\",\"note\":\"Journal Abbreviation: WWW '21\",\"keywords\":\"evaluation, information retrieval, fair ranking, fairness\",\"pages\":\"1065–1075\",\"bibtex\":\"@inproceedings{kirnap_estimation_2021,\\n\\taddress = {New York, NY, USA},\\n\\ttitle = {Estimation of fair ranking metrics with incomplete judgments},\\n\\turl = {https://arxiv.org/abs/2108.05152},\\n\\tdoi = {10.1145/3442381.3450080},\\n\\tabstract = {There is increasing attention to evaluating the fairness of search system\\nranking decisions. These metrics often consider the membership of items to\\nparticular groups, often identified using protected attributes such as\\ngender or ethnicity. To date, these metrics typically assume the\\navailability and completeness of protected attribute labels of items.\\nHowever, the protected attributes of individuals are rarely present,\\nlimiting the application of fair ranking metrics in large scale systems.\\nIn order to address this problem, we propose a sampling strategy and\\nestimation technique for four fair ranking metrics. We formulate a robust\\nand unbiased estimator which can operate even with very limited number of\\nlabeled items. We evaluate our approach using both simulated and real\\nworld data. Our experimental results demonstrate that our method can\\nestimate this family of fair ranking metrics and provides a robust,\\nreliable alternative to exhaustive or random data annotation.},\\n\\turldate = {2021-04-27},\\n\\tbooktitle = {{WWW} '21},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Kırnap, Ömer and Diaz, Fernando and Biega, Asia and Ekstrand, Michael and Carterette, Ben and Yilmaz, Emine},\\n\\tmonth = apr,\\n\\tyear = {2021},\\n\\tnote = {Journal Abbreviation: WWW '21},\\n\\tkeywords = {evaluation, information retrieval, fair ranking, fairness},\\n\\tpages = {1065--1075},\\n}\\n\\n\",\"author_short\":[\"Kırnap, Ö.\",\"Diaz, F.\",\"Biega, A.\",\"Ekstrand, M.\",\"Carterette, B.\",\"Yilmaz, E.\"],\"key\":\"kirnap_estimation_2021\",\"id\":\"kirnap_estimation_2021\",\"bibbaseid\":\"krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://arxiv.org/abs/2108.05152\"},\"keyword\":[\"evaluation\",\"information retrieval\",\"fair ranking\",\"fairness\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"title\":\"Exploring author gender in book rating and recommendation\",\"url\":\"https://dl.acm.org/doi/10.1145/3240323.3240373\",\"doi\":\"10.1145/3240323.3240373\",\"abstract\":\"Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as gender or ethnic discrimination in publishing. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\",\"publisher\":\"ACM\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Tian\"],\"firstnames\":[\"Mucun\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kazi\"],\"firstnames\":[\"Mohammed\",\"R\",\"Imran\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mehrpouyan\"],\"firstnames\":[\"Hoda\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kluver\"],\"firstnames\":[\"Daniel\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2018\",\"bibtex\":\"@inproceedings{ekstrand_exploring_2018,\\n\\taddress = {New York, NY, USA},\\n\\ttitle = {Exploring author gender in book rating and recommendation},\\n\\turl = {https://dl.acm.org/doi/10.1145/3240323.3240373},\\n\\tdoi = {10.1145/3240323.3240373},\\n\\tabstract = {Collaborative filtering algorithms find useful patterns in rating and\\nconsumption data and exploit these patterns to guide users to good items.\\nMany of the patterns in rating datasets reflect important real-world\\ndifferences between the various users and items in the data; other\\npatterns may be irrelevant or possibly undesirable for social or ethical\\nreasons, particularly if they reflect undesired discrimination, such as\\ngender or ethnic discrimination in publishing. In this work, we examine\\nthe response of collaborative filtering recommender algorithms to the\\ndistribution of their input data with respect to a dimension of social\\nconcern, namely content creator gender. Using publicly-available book\\nratings data, we measure the distribution of the genders of the authors of\\nbooks in user rating profiles and recommendation lists produced from this\\ndata. We find that common collaborative filtering algorithms differ in the\\ngender distribution of their recommendation lists, and in the relationship\\nof that output distribution to user profile distribution.},\\n\\tpublisher = {ACM},\\n\\tauthor = {Ekstrand, Michael D and Tian, Mucun and Kazi, Mohammed R Imran and Mehrpouyan, Hoda and Kluver, Daniel},\\n\\tmonth = sep,\\n\\tyear = {2018},\\n}\\n\\n\",\"author_short\":[\"Ekstrand, M. D\",\"Tian, M.\",\"Kazi, M. R I.\",\"Mehrpouyan, H.\",\"Kluver, D.\"],\"key\":\"ekstrand_exploring_2018\",\"id\":\"ekstrand_exploring_2018\",\"bibbaseid\":\"ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3240323.3240373\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":0,\"html\":\"\"},{\"bibtype\":\"misc\",\"type\":\"misc\",\"title\":\"Towards measuring fairness in grid layout in recommender systems\",\"copyright\":\"All rights reserved\",\"url\":\"http://arxiv.org/abs/2309.10271\",\"doi\":\"10.48550/arXiv.2309.10271\",\"abstract\":\"There has been significant research in the last five years on ensuring the providers of items in a recommender system are treated fairly, particularly in terms of the exposure the system provides to their work through its results. However, the metrics developed to date have all been designed and tested for linear ranked lists. It is unknown whether and how existing fair ranking metrics for linear layouts can be applied to grid-based displays. Moreover, depending on the device (phone, tab, or laptop) users use to interact with systems, column size is adjusted using column reduction approaches in a grid-view. The visibility or exposure of recommended items in grid layouts varies based on column sizes and column reduction approaches as well. In this paper, we extend existing fair ranking concepts and metrics to study provider-side group fairness in grid layouts, present an analysis of the behavior of these grid adaptations of fair ranking metrics, and study how their behavior changes across different grid ranking layout designs and geometries. We examine how fairness scores change with different ranking layouts to yield insights into (1) the consistency of fair ranking measurements across layouts; (2) whether rankings optimized for fairness in a linear ranking remain fair when the results are displayed in a grid; and (3) the impact of column reduction approaches to support different device geometries on fairness measurement. This work highlights the need to use layout-specific user attention models when measuring fairness of rankings, and provide practitioners with a first set of insights on what to expect when translating existing fair ranking metrics to the grid layouts in wide use today.\",\"urldate\":\"2023-11-16\",\"publisher\":\"arXiv\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2023\",\"note\":\"arXiv:2309.10271 [cs]\",\"keywords\":\"Computer Science - Information Retrieval\",\"bibtex\":\"@misc{raj_towards_2023,\\n\\ttitle = {Towards measuring fairness in grid layout in recommender systems},\\n\\tcopyright = {All rights reserved},\\n\\turl = {http://arxiv.org/abs/2309.10271},\\n\\tdoi = {10.48550/arXiv.2309.10271},\\n\\tabstract = {There has been significant research in the last five years on ensuring the providers of items in a recommender system are treated fairly, particularly in terms of the exposure the system provides to their work through its results. However, the metrics developed to date have all been designed and tested for linear ranked lists. It is unknown whether and how existing fair ranking metrics for linear layouts can be applied to grid-based displays. Moreover, depending on the device (phone, tab, or laptop) users use to interact with systems, column size is adjusted using column reduction approaches in a grid-view. The visibility or exposure of recommended items in grid layouts varies based on column sizes and column reduction approaches as well. In this paper, we extend existing fair ranking concepts and metrics to study provider-side group fairness in grid layouts, present an analysis of the behavior of these grid adaptations of fair ranking metrics, and study how their behavior changes across different grid ranking layout designs and geometries. We examine how fairness scores change with different ranking layouts to yield insights into (1) the consistency of fair ranking measurements across layouts; (2) whether rankings optimized for fairness in a linear ranking remain fair when the results are displayed in a grid; and (3) the impact of column reduction approaches to support different device geometries on fairness measurement. This work highlights the need to use layout-specific user attention models when measuring fairness of rankings, and provide practitioners with a first set of insights on what to expect when translating existing fair ranking metrics to the grid layouts in wide use today.},\\n\\turldate = {2023-11-16},\\n\\tpublisher = {arXiv},\\n\\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\\n\\tmonth = sep,\\n\\tyear = {2023},\\n\\tnote = {arXiv:2309.10271 [cs]},\\n\\tkeywords = {Computer Science - Information Retrieval},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Ekstrand, M. D.\"],\"key\":\"raj_towards_2023\",\"id\":\"raj_towards_2023\",\"bibbaseid\":\"raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2309.10271\"},\"keyword\":[\"Computer Science - Information Retrieval\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Exploring author gender in book rating and recommendation\",\"volume\":\"31\",\"issn\":\"0924-1868\",\"url\":\"https://md.ekstrandom.net/pubs/bag-extended\",\"doi\":\"10.1007/s11257-020-09284-2\",\"abstract\":\"Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\",\"number\":\"3\",\"urldate\":\"2020-06-05\",\"journal\":\"User Modeling and User-Adapted Interaction\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kluver\"],\"firstnames\":[\"Daniel\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2021\",\"pages\":\"377–420\",\"bibtex\":\"@article{ekstrand_exploring_2021,\\n\\ttitle = {Exploring author gender in book rating and recommendation},\\n\\tvolume = {31},\\n\\tissn = {0924-1868},\\n\\turl = {https://md.ekstrandom.net/pubs/bag-extended},\\n\\tdoi = {10.1007/s11257-020-09284-2},\\n\\tabstract = {Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.},\\n\\tnumber = {3},\\n\\turldate = {2020-06-05},\\n\\tjournal = {User Modeling and User-Adapted Interaction},\\n\\tauthor = {Ekstrand, Michael D and Kluver, Daniel},\\n\\tmonth = jul,\\n\\tyear = {2021},\\n\\tpages = {377--420},\\n}\\n\",\"author_short\":[\"Ekstrand, M. D\",\"Kluver, D.\"],\"key\":\"ekstrand_exploring_2021\",\"id\":\"ekstrand_exploring_2021\",\"bibbaseid\":\"ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/bag-extended\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":12,\"html\":\"\"}],\n      metadata: {\"authorlinks\":{}},\n      ownerID: undefined\n    };\n  </script>\n\n  <script type=\"text/javascript\">\n  var site$;\n  if (typeof $ != 'undefined') {\n    console.log('existing jquery: storing and then restoring');\n    site$ = $;\n    $ = null;\n  }\n  </script>\n\n  <script src=\"https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js\">\n  </script>\n  <script type=\"text/javascript\">\n  if (site$) {\n    console.log('existing jquery: avoiding conflict and restoring');\n    bb$ = $.noConflict(true);\n    $ = site$;\n  } else {\n    bb$ = $;\n  }\n  </script>\n\n  <script src=\"//bibbase.org/js/bibbase.min.js\"\n          type=\"text/javascript\">\n  </script>\n\n  <script type=\"text/javascript\"\n          src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML\">\n  </script>\n  <script type=\"text/x-mathjax-config\">\n    MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\\\(','\\\\)']]},\n                        skipTags: [\"body\"],\n                        processClass: \"bibbase_paper\"});\n  </script>\n\n  <style>\n  @media print {\n    .dontprint {\n        display: none !important;\n    }\n\n  .bibbase_group {\n    background-color: #E0E0E0;\n    font-style: italic;\n    font-size: larger;\n    text-shadow: 0px 0px 3px #FFFFFF;\n    border-radius: 4px 4px 4px 4px;\n    -moz-border-radius: 4px 4px 4px 4px;\n    -webkit-border-radius: 4px;\n    padding: 5px 40px 5px;\n    margin-top: 10px;\n    margin-bottom: 5px;\n    cursor: pointer;\n  }\n\n  .bibbase_paper {\n    margin-bottom: 5px;\n  }\n\n  }\n  </style>\n\n  \n  <div style=\"float: right; margin-right: 10px; margin-top: 10px; text-align: right; font-size: 12px\">\n    generated by\n    <a href=\"//bibbase.org\"\n       onclick=\"trackOutboundLink('bibbase', 'logo clicked', window.location.href, '//bibbase.org'); return false;\">\n      <img src=\"//bibbase.org/img/logo.svg\"\n           style=\"vertical-align: middle; margin-left: 3px; height: 16px;\"/\n           alt=\"bibbase.org\"\n           title=\"BibBase – The easiest way to maintain your publications page.\"\n        ></a>\n    \n  </div>\n  \n\n  <div id=\"bibbase_header\" class=\"dontprint\" style=\"\">\n\n    <ul class=\"nav nav-pills\" style=\"margin: 0px;\">\n\n      <li class=\"dropdown\" id=\"groupby_dropdown\">\n      <a class=\"dropdown-toggle\"\n         data-toggle=\"dropdown\"\n         href=\"#\">\n          Group by\n          <b class=\"caret\"></b>\n      </a>\n      <ul class=\"dropdown-menu\">\n        <li class=\"groupby year\"><a onclick=\"groupby('year')\">\n            Year</a>\n        </li>\n        <li class=\"groupby author\"><a onclick=\"groupby('author_short')\">\n            Author</a>\n        </li>\n        <li class=\"groupby type\"><a onclick=\"groupby('type')\">\n            Type</a>\n        </li>\n        <li class=\"groupby keyword\"><a onclick=\"groupby('keyword')\">\n            Keyword</a>\n        </li>\n        <li class=\"groupby downloads\"><a onclick=\"groupby('downloads')\">\n            Downloads</a>\n        </li>\n      </ul>\n      </li>\n\n\n      <li class=\"dropdown\" id=\"menu_dropdown\">\n      <a class=\"dropdown-toggle\"\n         data-toggle=\"dropdown\"\n         href=\"#\" alt=\"controls\">\n          <i class=\"fa fa-reorder\" alt=\"controls\"></i>\n          <b class=\"caret\"></b>\n      </a>\n      <ul class=\"dropdown-menu\">\n        <li>\n          <a onclick=\"toggleAll()\">\n            <i class=\"fa fa-chevron-down fa-fw\"></i> Expand/Collapse All\n          </a>\n        </li>\n        <li>\n          <a download=\"items\" href=\"data:text/bibtex;base64,QGlucHJvY2VlZGluZ3N7cmFqX21lYXN1cmluZ18yMDIyLAoJdGl0bGUgPSB7TWVhc3VyaW5nIGZhaXJuZXNzIGluIHJhbmtlZCByZXN1bHRzOiBhbiBhbmFseXRpY2FsIGFuZCBlbXBpcmljYWwgY29tcGFyaXNvbn0sCgl1cmwgPSB7aHR0cHM6Ly9tZC5la3N0cmFuZG9tLm5ldC9wdWJzL2ZhaXItcmFua2luZ30sCglkb2kgPSB7MTAuMTE0NS8zNDc3NDk1LjM1MzIwMTh9LAoJYWJzdHJhY3QgPSB7SW5mb3JtYXRpb24gYWNjZXNzIHN5c3RlbXMsIHN1Y2ggYXMgc2VhcmNoIGFuZCByZWNvbW1lbmRlciBzeXN0ZW1zLCBvZnRlbiB1c2UgcmFua2VkIGxpc3RzIHRvIHByZXNlbnQgcmVzdWx0cyBiZWxpZXZlZCB0byBiZSByZWxldmFudCB0byB0aGUgdXNlcidzIGluZm9ybWF0aW9uIG5lZWQuIEV2YWx1YXRpbmcgdGhlc2UgbGlzdHMgZm9yIHRoZWlyIGZhaXJuZXNzIGFsb25nIHdpdGggb3RoZXIgdHJhZGl0aW9uYWwgbWV0cmljcyBwcm92aWRlcyBhIG1vcmUgY29tcGxldGUgdW5kZXJzdGFuZGluZyBvZiBhbiBpbmZvcm1hdGlvbiBhY2Nlc3Mgc3lzdGVtJ3MgYmVoYXZpb3IgYmV5b25kIGFjY3VyYWN5IG9yIHV0aWxpdHkgY29uc3RydWN0cy4gVG8gbWVhc3VyZSB0aGUgKHVuKWZhaXJuZXNzIG9mIHJhbmtpbmdzLCBwYXJ0aWN1bGFybHkgd2l0aCByZXNwZWN0IHRvIHRoZSBwcm90ZWN0ZWQgZ3JvdXAocykgb2YgcHJvZHVjZXJzIG9yIHByb3ZpZGVycywgc2V2ZXJhbCBtZXRyaWNzIGhhdmUgYmVlbiBwcm9wb3NlZCBpbiB0aGUgbGFzdCBzZXZlcmFsIHllYXJzLiBIb3dldmVyLCBhbiBlbXBpcmljYWwgYW5kIGNvbXBhcmF0aXZlIGFuYWx5c2VzIG9mIHRoZXNlIG1ldHJpY3Mgc2hvd2luZyB0aGUgYXBwbGljYWJpbGl0eSB0byBzcGVjaWZpYyBzY2VuYXJpbyBvciByZWFsIGRhdGEsIGNvbmNlcHR1YWwgc2ltaWxhcml0aWVzLCBhbmQgZGlmZmVyZW5jZXMgaXMgc3RpbGwgbGFja2luZy4KCldlIGFpbSB0byBicmlkZ2UgdGhlIGdhcCBiZXR3ZWVuIHRoZW9yZXRpY2FsIGFuZCBwcmFjdGljYWwgYXAtcGxpY2F0aW9uIG9mIHRoZXNlIG1ldHJpY3MuIEluIHRoaXMgcGFwZXIgd2UgZGVzY3JpYmUgc2V2ZXJhbCBmYWlyIHJhbmtpbmcgbWV0cmljcyBmcm9tIHRoZSBleGlzdGluZyBsaXRlcmF0dXJlIGluIGEgY29tbW9uIG5vdGF0aW9uLCBlbmFibGluZyBkaXJlY3QgY29tcGFyaXNvbiBvZiB0aGVpciBhcHByb2FjaGVzIGFuZCBhc3N1bXB0aW9ucywgYW5kIGVtcGlyaWNhbGx5IGNvbXBhcmUgdGhlbSBvbiB0aGUgc2FtZSBleHBlcmltZW50YWwgc2V0dXAgYW5kIGRhdGEgc2V0cyBpbiB0aGUgY29udGV4dCBvZiB0aHJlZSBpbmZvcm1hdGlvbiBhY2Nlc3MgdGFza3MuIFdlIGFsc28gcHJvdmlkZSBhIHNlbnNpdGl2aXR5IGFuYWx5c2lzIHRvIGFzc2VzcyB0aGUgaW1wYWN0IG9mIHRoZSBkZXNpZ24gY2hvaWNlcyBhbmQgcGFyYW1ldGVyIHNldHRpbmdzIHRoYXQgZ28gaW4gdG8gdGhlc2UgbWV0cmljcyBhbmQgcG9pbnQgdG8gYWRkaXRpb25hbCB3b3JrIG5lZWRlZCB0byBpbXByb3ZlIGZhaXJuZXNzIG1lYXN1cmVtZW50Ln0sCglib29rdGl0bGUgPSB7UHJvY2VlZGluZ3Mgb2YgdGhlIDQ1dGgge0ludGVybmF0aW9uYWx9IHtBQ019IHtTSUdJUn0ge0NvbmZlcmVuY2V9IG9uIHtSZXNlYXJjaH0gYW5kIHtEZXZlbG9wbWVudH0gaW4ge0luZm9ybWF0aW9ufSB7UmV0cmlldmFsfX0sCglwdWJsaXNoZXIgPSB7QUNNfSwKCWF1dGhvciA9IHtSYWosIEFtaWZhIGFuZCBFa3N0cmFuZCwgTWljaGFlbCBEfSwKCW1vbnRoID0ganVsLAoJeWVhciA9IHsyMDIyfSwKCXBhZ2VzID0gezcyNi0tNzM2fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tyYWpfdG93YXJkc18yMDI0LAoJc2VyaWVzID0ge3tMTkNTfX0sCgl0aXRsZSA9IHtUb3dhcmRzIG9wdGltaXppbmcgcmFua2luZyBpbiBncmlkLWxheW91dCBmb3IgcHJvdmlkZXItc2lkZSBmYWlybmVzc30sCgl2b2x1bWUgPSB7MTQ2MTJ9LAoJY29weXJpZ2h0ID0ge0FsbCByaWdodHMgcmVzZXJ2ZWR9LAoJdXJsID0ge2h0dHBzOi8vbWQuZWtzdHJhbmRvbS5uZXQvcHVicy9lY2lyLWZhaXItZ3JpZHN9LAoJZG9pID0gezEwLjEwMDcvOTc4LTMtMDMxLTU2MDY5LTlfN30sCglhYnN0cmFjdCA9IHtJbmZvcm1hdGlvbiBhY2Nlc3Mgc3lzdGVtcywgc3VjaCBhcyBzZWFyY2ggZW5naW5lcyBhbmQgcmVjb21tZW5kZXIgc3lzdGVtcywgb3JkZXIgYW5kIHBvc2l0aW9uIHJlc3VsdHMgYmFzZWQgb24gdGhlaXIgZXN0aW1hdGVkIHJlbGV2YW5jZS4gVGhlc2UgcmVzdWx0cyBhcmUgdGhlbiBldmFsdWF0ZWQgZm9yIGEgcmFuZ2Ugb2YgY29uY2VybnMsIGluY2x1ZGluZyBwcm92aWRlci1zaWRlIGZhaXJuZXNzOiB3aGV0aGVyIGV4cG9zdXJlIHRvIHVzZXJzIGlzIGZhaXJseSBkaXN0cmlidXRlZCBhbW9uZyBpdGVtcyBhbmQgdGhlIHBlb3BsZSB3aG8gY3JlYXRlZCB0aGVtLiBTZXZlcmFsIGZhaXJuZXNzLWF3YXJlIHJhbmtpbmcgYW5kIHJlLXJhbmtpbmcgdGVjaG5pcXVlcyBoYXZlIGJlZW4gcHJvcG9zZWQgdG8gZW5zdXJlIGZhaXIgZXhwb3N1cmUgZm9yIHByb3ZpZGVycywgYnV0IHRoaXMgd29yayBmb2N1c2VzIGFsbW9zdCBleGNsdXNpdmVseSBvbiBsaW5lYXIgbGF5b3V0cyBpbiB3aGljaCBpdGVtcyBhcmUgZGlzcGxheWVkIGluIHNpbmdsZSByYW5rZWQgbGlzdC4gTWFueSB3aWRlbHktdXNlZCBzeXN0ZW1zIHVzZSBvdGhlciBsYXlvdXRzLCBzdWNoIGFzIHRoZSBncmlkIHZpZXdzIGNvbW1vbiBpbiBzdHJlYW1pbmcgcGxhdGZvcm1zLCBpbWFnZSBzZWFyY2gsIGFuZCBvdGhlciBhcHBsaWNhdGlvbnMuIFByb3ZpZGluZyBmYWlyIGV4cG9zdXJlIHRvIHByb3ZpZGVycyBpbiBzdWNoIGxheW91dHMgaXMgbm90IHdlbGwtc3R1ZGllZC4gV2Ugc2VlayB0byBmaWxsIHRoaXMgZ2FwIGJ5IHByb3ZpZGluZyBhIGdyaWQtYXdhcmUgcmUtcmFua2luZyBhbGdvcml0aG0gdG8gb3B0aW1pemUgbGF5b3V0cyBmb3IgcHJvdmlkZXItc2lkZSBmYWlybmVzcyBieSBhZGFwdGluZyBleGlzdGluZyByZS1yYW5raW5nIHRlY2huaXF1ZXMgdG8gZ3JpZC1hd2FyZSBicm93c2luZyBtb2RlbHMsIGFuZCBhbiBhbmFseXNpcyBvZiB0aGUgZWZmZWN0IG9mIGdyaWQtc3BlY2lmaWMgZmFjdG9ycyBzdWNoIGFzIGRldmljZSBzaXplIG9uIHRoZSByZXN1bHRpbmcgZmFpcm5lc3Mgb3B0aW1pemF0aW9uLn0sCglsYW5ndWFnZSA9IHtlbn0sCgl1cmxkYXRlID0gezIwMjQtMDEtMDR9LAoJYm9va3RpdGxlID0ge0FkdmFuY2VzIGluIHtJbmZvcm1hdGlvbn0ge1JldHJpZXZhbH19LAoJcHVibGlzaGVyID0ge1NwcmluZ2VyfSwKCWF1dGhvciA9IHtSYWosIEFtaWZhIGFuZCBFa3N0cmFuZCwgTWljaGFlbCBELn0sCgltb250aCA9IG1hciwKCXllYXIgPSB7MjAyNH0sCglwYWdlcyA9IHs5MC0tMTA1fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tyYWpfY29tcGFyaW5nXzIwMjAsCgl0aXRsZSA9IHtDb21wYXJpbmcgZmFpciByYW5raW5nIG1ldHJpY3N9LAoJdXJsID0ge2h0dHA6Ly9hcnhpdi5vcmcvYWJzLzIwMDkuMDEzMTF9LAoJYWJzdHJhY3QgPSB7UmFua2luZyBpcyBhIGZ1bmRhbWVudGFsIGFzcGVjdCBvZiByZWNvbW1lbmRlciBzeXN0ZW1zLiBIb3dldmVyLCByYW5rZWQKb3V0cHV0cyBjYW4gYmUgc3VzY2VwdGlibGUgdG8gdmFyaW91cyBiaWFzZXM7IHNvbWUgb2YgdGhlc2UgbWF5IGNhdXNlCmRpc2FkdmFudGFnZXMgdG8gbWVtYmVycyBvZiBwcm90ZWN0ZWQgZ3JvdXBzLiBTZXZlcmFsIG1ldHJpY3MgaGF2ZSBiZWVuCnByb3Bvc2VkIHRvIHF1YW50aWZ5IHRoZSAodW4pZmFpcm5lc3Mgb2YgcmFua2luZ3MsIGJ1dCB0aGVyZSBoYXMgbm90IGJlZW4KdG8gZGF0ZSBhbnkgZGlyZWN0IGNvbXBhcmlzb24gb2YgdGhlc2UgbWV0cmljcy4gVGhpcyBjb21wbGljYXRlcyBkZWNpZGluZwp3aGF0IGZhaXJuZXNzIG1ldHJpY3MgYXJlIGFwcGxpY2FibGUgZm9yIHNwZWNpZmljIHNjZW5hcmlvcywgYW5kIGFzc2Vzc2luZwp0aGUgZXh0ZW50IHRvIHdoaWNoIG1ldHJpY3MgYWdyZWUgb3IgZGlzYWdyZWUuIEluIHRoaXMgcGFwZXIsIHdlIGRlc2NyaWJlCnNldmVyYWwgZmFpciByYW5raW5nIG1ldHJpY3MgaW4gYSBjb21tb24gbm90YXRpb24sIGVuYWJsaW5nIGRpcmVjdApjb21wYXJpc29uIG9mIHRoZWlyIGFwcHJvYWNoZXMgYW5kIGFzc3VtcHRpb25zLCBhbmQgZW1waXJpY2FsbHkgY29tcGFyZQp0aGVtIG9uIHRoZSBzYW1lIGV4cGVyaW1lbnRhbCBzZXR1cCBhbmQgZGF0YSBzZXQuIE91ciB3b3JrIHByb3ZpZGVzIGEKZGlyZWN0IGNvbXBhcmF0aXZlIGFuYWx5c2lzIGlkZW50aWZ5aW5nIHNpbWlsYXJpdGllcyBhbmQgZGlmZmVyZW5jZXMgb2YKZmFpciByYW5raW5nIG1ldHJpY3Mgc2VsZWN0ZWQgZm9yIG91ciB3b3JrLn0sCglhdXRob3IgPSB7UmFqLCBBbWlmYSBhbmQgV29vZCwgQ29ubm9yIGFuZCBNb250b2x5LCBBbmFuZGEgYW5kIEVrc3RyYW5kLCBNaWNoYWVsIER9LAoJbW9udGggPSBzZXAsCgl5ZWFyID0gezIwMjB9LAp9CgoKCkBpbnByb2NlZWRpbmdze2tpcm5hcF9lc3RpbWF0aW9uXzIwMjEsCglhZGRyZXNzID0ge05ldyBZb3JrLCBOWSwgVVNBfSwKCXRpdGxlID0ge0VzdGltYXRpb24gb2YgZmFpciByYW5raW5nIG1ldHJpY3Mgd2l0aCBpbmNvbXBsZXRlIGp1ZGdtZW50c30sCgl1cmwgPSB7aHR0cHM6Ly9hcnhpdi5vcmcvYWJzLzIxMDguMDUxNTJ9LAoJZG9pID0gezEwLjExNDUvMzQ0MjM4MS4zNDUwMDgwfSwKCWFic3RyYWN0ID0ge1RoZXJlIGlzIGluY3JlYXNpbmcgYXR0ZW50aW9uIHRvIGV2YWx1YXRpbmcgdGhlIGZhaXJuZXNzIG9mIHNlYXJjaCBzeXN0ZW0KcmFua2luZyBkZWNpc2lvbnMuIFRoZXNlIG1ldHJpY3Mgb2Z0ZW4gY29uc2lkZXIgdGhlIG1lbWJlcnNoaXAgb2YgaXRlbXMgdG8KcGFydGljdWxhciBncm91cHMsIG9mdGVuIGlkZW50aWZpZWQgdXNpbmcgcHJvdGVjdGVkIGF0dHJpYnV0ZXMgc3VjaCBhcwpnZW5kZXIgb3IgZXRobmljaXR5LiBUbyBkYXRlLCB0aGVzZSBtZXRyaWNzIHR5cGljYWxseSBhc3N1bWUgdGhlCmF2YWlsYWJpbGl0eSBhbmQgY29tcGxldGVuZXNzIG9mIHByb3RlY3RlZCBhdHRyaWJ1dGUgbGFiZWxzIG9mIGl0ZW1zLgpIb3dldmVyLCB0aGUgcHJvdGVjdGVkIGF0dHJpYnV0ZXMgb2YgaW5kaXZpZHVhbHMgYXJlIHJhcmVseSBwcmVzZW50LApsaW1pdGluZyB0aGUgYXBwbGljYXRpb24gb2YgZmFpciByYW5raW5nIG1ldHJpY3MgaW4gbGFyZ2Ugc2NhbGUgc3lzdGVtcy4KSW4gb3JkZXIgdG8gYWRkcmVzcyB0aGlzIHByb2JsZW0sIHdlIHByb3Bvc2UgYSBzYW1wbGluZyBzdHJhdGVneSBhbmQKZXN0aW1hdGlvbiB0ZWNobmlxdWUgZm9yIGZvdXIgZmFpciByYW5raW5nIG1ldHJpY3MuIFdlIGZvcm11bGF0ZSBhIHJvYnVzdAphbmQgdW5iaWFzZWQgZXN0aW1hdG9yIHdoaWNoIGNhbiBvcGVyYXRlIGV2ZW4gd2l0aCB2ZXJ5IGxpbWl0ZWQgbnVtYmVyIG9mCmxhYmVsZWQgaXRlbXMuIFdlIGV2YWx1YXRlIG91ciBhcHByb2FjaCB1c2luZyBib3RoIHNpbXVsYXRlZCBhbmQgcmVhbAp3b3JsZCBkYXRhLiBPdXIgZXhwZXJpbWVudGFsIHJlc3VsdHMgZGVtb25zdHJhdGUgdGhhdCBvdXIgbWV0aG9kIGNhbgplc3RpbWF0ZSB0aGlzIGZhbWlseSBvZiBmYWlyIHJhbmtpbmcgbWV0cmljcyBhbmQgcHJvdmlkZXMgYSByb2J1c3QsCnJlbGlhYmxlIGFsdGVybmF0aXZlIHRvIGV4aGF1c3RpdmUgb3IgcmFuZG9tIGRhdGEgYW5ub3RhdGlvbi59LAoJdXJsZGF0ZSA9IHsyMDIxLTA0LTI3fSwKCWJvb2t0aXRsZSA9IHt7V1dXfSAnMjF9LAoJcHVibGlzaGVyID0ge0Fzc29jaWF0aW9uIGZvciBDb21wdXRpbmcgTWFjaGluZXJ5fSwKCWF1dGhvciA9IHtLxLFybmFwLCDDlm1lciBhbmQgRGlheiwgRmVybmFuZG8gYW5kIEJpZWdhLCBBc2lhIGFuZCBFa3N0cmFuZCwgTWljaGFlbCBhbmQgQ2FydGVyZXR0ZSwgQmVuIGFuZCBZaWxtYXosIEVtaW5lfSwKCW1vbnRoID0gYXByLAoJeWVhciA9IHsyMDIxfSwKCW5vdGUgPSB7Sm91cm5hbCBBYmJyZXZpYXRpb246IFdXVyAnMjF9LAoJa2V5d29yZHMgPSB7ZXZhbHVhdGlvbiwgaW5mb3JtYXRpb24gcmV0cmlldmFsLCBmYWlyIHJhbmtpbmcsIGZhaXJuZXNzfSwKCXBhZ2VzID0gezEwNjUtLTEwNzV9LAp9CgoKCkBpbnByb2NlZWRpbmdze2Vrc3RyYW5kX2V4cGxvcmluZ18yMDE4LAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCgl0aXRsZSA9IHtFeHBsb3JpbmcgYXV0aG9yIGdlbmRlciBpbiBib29rIHJhdGluZyBhbmQgcmVjb21tZW5kYXRpb259LAoJdXJsID0ge2h0dHBzOi8vZGwuYWNtLm9yZy9kb2kvMTAuMTE0NS8zMjQwMzIzLjMyNDAzNzN9LAoJZG9pID0gezEwLjExNDUvMzI0MDMyMy4zMjQwMzczfSwKCWFic3RyYWN0ID0ge0NvbGxhYm9yYXRpdmUgZmlsdGVyaW5nIGFsZ29yaXRobXMgZmluZCB1c2VmdWwgcGF0dGVybnMgaW4gcmF0aW5nIGFuZApjb25zdW1wdGlvbiBkYXRhIGFuZCBleHBsb2l0IHRoZXNlIHBhdHRlcm5zIHRvIGd1aWRlIHVzZXJzIHRvIGdvb2QgaXRlbXMuCk1hbnkgb2YgdGhlIHBhdHRlcm5zIGluIHJhdGluZyBkYXRhc2V0cyByZWZsZWN0IGltcG9ydGFudCByZWFsLXdvcmxkCmRpZmZlcmVuY2VzIGJldHdlZW4gdGhlIHZhcmlvdXMgdXNlcnMgYW5kIGl0ZW1zIGluIHRoZSBkYXRhOyBvdGhlcgpwYXR0ZXJucyBtYXkgYmUgaXJyZWxldmFudCBvciBwb3NzaWJseSB1bmRlc2lyYWJsZSBmb3Igc29jaWFsIG9yIGV0aGljYWwKcmVhc29ucywgcGFydGljdWxhcmx5IGlmIHRoZXkgcmVmbGVjdCB1bmRlc2lyZWQgZGlzY3JpbWluYXRpb24sIHN1Y2ggYXMKZ2VuZGVyIG9yIGV0aG5pYyBkaXNjcmltaW5hdGlvbiBpbiBwdWJsaXNoaW5nLiBJbiB0aGlzIHdvcmssIHdlIGV4YW1pbmUKdGhlIHJlc3BvbnNlIG9mIGNvbGxhYm9yYXRpdmUgZmlsdGVyaW5nIHJlY29tbWVuZGVyIGFsZ29yaXRobXMgdG8gdGhlCmRpc3RyaWJ1dGlvbiBvZiB0aGVpciBpbnB1dCBkYXRhIHdpdGggcmVzcGVjdCB0byBhIGRpbWVuc2lvbiBvZiBzb2NpYWwKY29uY2VybiwgbmFtZWx5IGNvbnRlbnQgY3JlYXRvciBnZW5kZXIuIFVzaW5nIHB1YmxpY2x5LWF2YWlsYWJsZSBib29rCnJhdGluZ3MgZGF0YSwgd2UgbWVhc3VyZSB0aGUgZGlzdHJpYnV0aW9uIG9mIHRoZSBnZW5kZXJzIG9mIHRoZSBhdXRob3JzIG9mCmJvb2tzIGluIHVzZXIgcmF0aW5nIHByb2ZpbGVzIGFuZCByZWNvbW1lbmRhdGlvbiBsaXN0cyBwcm9kdWNlZCBmcm9tIHRoaXMKZGF0YS4gV2UgZmluZCB0aGF0IGNvbW1vbiBjb2xsYWJvcmF0aXZlIGZpbHRlcmluZyBhbGdvcml0aG1zIGRpZmZlciBpbiB0aGUKZ2VuZGVyIGRpc3RyaWJ1dGlvbiBvZiB0aGVpciByZWNvbW1lbmRhdGlvbiBsaXN0cywgYW5kIGluIHRoZSByZWxhdGlvbnNoaXAKb2YgdGhhdCBvdXRwdXQgZGlzdHJpYnV0aW9uIHRvIHVzZXIgcHJvZmlsZSBkaXN0cmlidXRpb24ufSwKCXB1Ymxpc2hlciA9IHtBQ019LAoJYXV0aG9yID0ge0Vrc3RyYW5kLCBNaWNoYWVsIEQgYW5kIFRpYW4sIE11Y3VuIGFuZCBLYXppLCBNb2hhbW1lZCBSIEltcmFuIGFuZCBNZWhycG91eWFuLCBIb2RhIGFuZCBLbHV2ZXIsIERhbmllbH0sCgltb250aCA9IHNlcCwKCXllYXIgPSB7MjAxOH0sCn0KCgoKQG1pc2N7cmFqX3Rvd2FyZHNfMjAyMywKCXRpdGxlID0ge1Rvd2FyZHMgbWVhc3VyaW5nIGZhaXJuZXNzIGluIGdyaWQgbGF5b3V0IGluIHJlY29tbWVuZGVyIHN5c3RlbXN9LAoJY29weXJpZ2h0ID0ge0FsbCByaWdodHMgcmVzZXJ2ZWR9LAoJdXJsID0ge2h0dHA6Ly9hcnhpdi5vcmcvYWJzLzIzMDkuMTAyNzF9LAoJZG9pID0gezEwLjQ4NTUwL2FyWGl2LjIzMDkuMTAyNzF9LAoJYWJzdHJhY3QgPSB7VGhlcmUgaGFzIGJlZW4gc2lnbmlmaWNhbnQgcmVzZWFyY2ggaW4gdGhlIGxhc3QgZml2ZSB5ZWFycyBvbiBlbnN1cmluZyB0aGUgcHJvdmlkZXJzIG9mIGl0ZW1zIGluIGEgcmVjb21tZW5kZXIgc3lzdGVtIGFyZSB0cmVhdGVkIGZhaXJseSwgcGFydGljdWxhcmx5IGluIHRlcm1zIG9mIHRoZSBleHBvc3VyZSB0aGUgc3lzdGVtIHByb3ZpZGVzIHRvIHRoZWlyIHdvcmsgdGhyb3VnaCBpdHMgcmVzdWx0cy4gSG93ZXZlciwgdGhlIG1ldHJpY3MgZGV2ZWxvcGVkIHRvIGRhdGUgaGF2ZSBhbGwgYmVlbiBkZXNpZ25lZCBhbmQgdGVzdGVkIGZvciBsaW5lYXIgcmFua2VkIGxpc3RzLiBJdCBpcyB1bmtub3duIHdoZXRoZXIgYW5kIGhvdyBleGlzdGluZyBmYWlyIHJhbmtpbmcgbWV0cmljcyBmb3IgbGluZWFyIGxheW91dHMgY2FuIGJlIGFwcGxpZWQgdG8gZ3JpZC1iYXNlZCBkaXNwbGF5cy4gTW9yZW92ZXIsIGRlcGVuZGluZyBvbiB0aGUgZGV2aWNlIChwaG9uZSwgdGFiLCBvciBsYXB0b3ApIHVzZXJzIHVzZSB0byBpbnRlcmFjdCB3aXRoIHN5c3RlbXMsIGNvbHVtbiBzaXplIGlzIGFkanVzdGVkIHVzaW5nIGNvbHVtbiByZWR1Y3Rpb24gYXBwcm9hY2hlcyBpbiBhIGdyaWQtdmlldy4gVGhlIHZpc2liaWxpdHkgb3IgZXhwb3N1cmUgb2YgcmVjb21tZW5kZWQgaXRlbXMgaW4gZ3JpZCBsYXlvdXRzIHZhcmllcyBiYXNlZCBvbiBjb2x1bW4gc2l6ZXMgYW5kIGNvbHVtbiByZWR1Y3Rpb24gYXBwcm9hY2hlcyBhcyB3ZWxsLiBJbiB0aGlzIHBhcGVyLCB3ZSBleHRlbmQgZXhpc3RpbmcgZmFpciByYW5raW5nIGNvbmNlcHRzIGFuZCBtZXRyaWNzIHRvIHN0dWR5IHByb3ZpZGVyLXNpZGUgZ3JvdXAgZmFpcm5lc3MgaW4gZ3JpZCBsYXlvdXRzLCBwcmVzZW50IGFuIGFuYWx5c2lzIG9mIHRoZSBiZWhhdmlvciBvZiB0aGVzZSBncmlkIGFkYXB0YXRpb25zIG9mIGZhaXIgcmFua2luZyBtZXRyaWNzLCBhbmQgc3R1ZHkgaG93IHRoZWlyIGJlaGF2aW9yIGNoYW5nZXMgYWNyb3NzIGRpZmZlcmVudCBncmlkIHJhbmtpbmcgbGF5b3V0IGRlc2lnbnMgYW5kIGdlb21ldHJpZXMuIFdlIGV4YW1pbmUgaG93IGZhaXJuZXNzIHNjb3JlcyBjaGFuZ2Ugd2l0aCBkaWZmZXJlbnQgcmFua2luZyBsYXlvdXRzIHRvIHlpZWxkIGluc2lnaHRzIGludG8gKDEpIHRoZSBjb25zaXN0ZW5jeSBvZiBmYWlyIHJhbmtpbmcgbWVhc3VyZW1lbnRzIGFjcm9zcyBsYXlvdXRzOyAoMikgd2hldGhlciByYW5raW5ncyBvcHRpbWl6ZWQgZm9yIGZhaXJuZXNzIGluIGEgbGluZWFyIHJhbmtpbmcgcmVtYWluIGZhaXIgd2hlbiB0aGUgcmVzdWx0cyBhcmUgZGlzcGxheWVkIGluIGEgZ3JpZDsgYW5kICgzKSB0aGUgaW1wYWN0IG9mIGNvbHVtbiByZWR1Y3Rpb24gYXBwcm9hY2hlcyB0byBzdXBwb3J0IGRpZmZlcmVudCBkZXZpY2UgZ2VvbWV0cmllcyBvbiBmYWlybmVzcyBtZWFzdXJlbWVudC4gVGhpcyB3b3JrIGhpZ2hsaWdodHMgdGhlIG5lZWQgdG8gdXNlIGxheW91dC1zcGVjaWZpYyB1c2VyIGF0dGVudGlvbiBtb2RlbHMgd2hlbiBtZWFzdXJpbmcgZmFpcm5lc3Mgb2YgcmFua2luZ3MsIGFuZCBwcm92aWRlIHByYWN0aXRpb25lcnMgd2l0aCBhIGZpcnN0IHNldCBvZiBpbnNpZ2h0cyBvbiB3aGF0IHRvIGV4cGVjdCB3aGVuIHRyYW5zbGF0aW5nIGV4aXN0aW5nIGZhaXIgcmFua2luZyBtZXRyaWNzIHRvIHRoZSBncmlkIGxheW91dHMgaW4gd2lkZSB1c2UgdG9kYXkufSwKCXVybGRhdGUgPSB7MjAyMy0xMS0xNn0sCglwdWJsaXNoZXIgPSB7YXJYaXZ9LAoJYXV0aG9yID0ge1JhaiwgQW1pZmEgYW5kIEVrc3RyYW5kLCBNaWNoYWVsIEQufSwKCW1vbnRoID0gc2VwLAoJeWVhciA9IHsyMDIzfSwKCW5vdGUgPSB7YXJYaXY6MjMwOS4xMDI3MSBbY3NdfSwKCWtleXdvcmRzID0ge0NvbXB1dGVyIFNjaWVuY2UgLSBJbmZvcm1hdGlvbiBSZXRyaWV2YWx9LAp9CgoKCkBhcnRpY2xle2Vrc3RyYW5kX2V4cGxvcmluZ18yMDIxLAoJdGl0bGUgPSB7RXhwbG9yaW5nIGF1dGhvciBnZW5kZXIgaW4gYm9vayByYXRpbmcgYW5kIHJlY29tbWVuZGF0aW9ufSwKCXZvbHVtZSA9IHszMX0sCglpc3NuID0gezA5MjQtMTg2OH0sCgl1cmwgPSB7aHR0cHM6Ly9tZC5la3N0cmFuZG9tLm5ldC9wdWJzL2JhZy1leHRlbmRlZH0sCglkb2kgPSB7MTAuMTAwNy9zMTEyNTctMDIwLTA5Mjg0LTJ9LAoJYWJzdHJhY3QgPSB7Q29sbGFib3JhdGl2ZSBmaWx0ZXJpbmcgYWxnb3JpdGhtcyBmaW5kIHVzZWZ1bCBwYXR0ZXJucyBpbiByYXRpbmcgYW5kIGNvbnN1bXB0aW9uIGRhdGEgYW5kIGV4cGxvaXQgdGhlc2UgcGF0dGVybnMgdG8gZ3VpZGUgdXNlcnMgdG8gZ29vZCBpdGVtcy4gTWFueSBvZiB0aGUgcGF0dGVybnMgaW4gcmF0aW5nIGRhdGFzZXRzIHJlZmxlY3QgaW1wb3J0YW50IHJlYWwtd29ybGQgZGlmZmVyZW5jZXMgYmV0d2VlbiB0aGUgdmFyaW91cyB1c2VycyBhbmQgaXRlbXMgaW4gdGhlIGRhdGE7IG90aGVyIHBhdHRlcm5zIG1heSBiZSBpcnJlbGV2YW50IG9yIHBvc3NpYmx5IHVuZGVzaXJhYmxlIGZvciBzb2NpYWwgb3IgZXRoaWNhbCByZWFzb25zLCBwYXJ0aWN1bGFybHkgaWYgdGhleSByZWZsZWN0IHVuZGVzaXJlZCBkaXNjcmltaW5hdGlvbiwgc3VjaCBhcyBkaXNjcmltaW5hdGlvbiBpbiBwdWJsaXNoaW5nIG9yIHB1cmNoYXNpbmcgYWdhaW5zdCBhdXRob3JzIHdobyBhcmUgd29tZW4gb3IgZXRobmljIG1pbm9yaXRpZXMuIEluIHRoaXMgd29yaywgd2UgZXhhbWluZSB0aGUgcmVzcG9uc2Ugb2YgY29sbGFib3JhdGl2ZSBmaWx0ZXJpbmcgcmVjb21tZW5kZXIgYWxnb3JpdGhtcyB0byB0aGUgZGlzdHJpYnV0aW9uIG9mIHRoZWlyIGlucHV0IGRhdGEgd2l0aCByZXNwZWN0IHRvIGEgZGltZW5zaW9uIG9mIHNvY2lhbCBjb25jZXJuLCBuYW1lbHkgY29udGVudCBjcmVhdG9yIGdlbmRlci4gVXNpbmcgcHVibGljbHktYXZhaWxhYmxlIGJvb2sgcmF0aW5ncyBkYXRhLCB3ZSBtZWFzdXJlIHRoZSBkaXN0cmlidXRpb24gb2YgdGhlIGdlbmRlcnMgb2YgdGhlIGF1dGhvcnMgb2YgYm9va3MgaW4gdXNlciByYXRpbmcgcHJvZmlsZXMgYW5kIHJlY29tbWVuZGF0aW9uIGxpc3RzIHByb2R1Y2VkIGZyb20gdGhpcyBkYXRhLiBXZSBmaW5kIHRoYXQgY29tbW9uIGNvbGxhYm9yYXRpdmUgZmlsdGVyaW5nIGFsZ29yaXRobXMgZGlmZmVyIGluIHRoZSBnZW5kZXIgZGlzdHJpYnV0aW9uIG9mIHRoZWlyIHJlY29tbWVuZGF0aW9uIGxpc3RzLCBhbmQgaW4gdGhlIHJlbGF0aW9uc2hpcCBvZiB0aGF0IG91dHB1dCBkaXN0cmlidXRpb24gdG8gdXNlciBwcm9maWxlIGRpc3RyaWJ1dGlvbi59LAoJbnVtYmVyID0gezN9LAoJdXJsZGF0ZSA9IHsyMDIwLTA2LTA1fSwKCWpvdXJuYWwgPSB7VXNlciBNb2RlbGluZyBhbmQgVXNlci1BZGFwdGVkIEludGVyYWN0aW9ufSwKCWF1dGhvciA9IHtFa3N0cmFuZCwgTWljaGFlbCBEIGFuZCBLbHV2ZXIsIERhbmllbH0sCgltb250aCA9IGp1bCwKCXllYXIgPSB7MjAyMX0sCglwYWdlcyA9IHszNzctLTQyMH0sCn0K\">\n            <i class=\"fa fa-download fa-fw\"></i> Download BibTeX\n          </a>\n        </li>\n        <li>\n          <a href=\"//bibbase.org/rss?bib=https://api.zotero.org/users/6655/collections/7SR56WJZ/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&amp;format=bibtex&amp;limit=100\">\n            <i class=\"fa fa-rss fa-fw\"></i> RSS Feed\n          </a>\n          </li>\n      </ul>\n      </li>\n\n      \n\n\n      \n    </ul>\n\n\n    <div class=\"alert alert-success bibbase_msg\" id=\"bibbase_msg_embed\"\n         style=\"display: none;\">\n\n      Excellent! Next you can\n      <a href=\"/network/register?next=/network/newSiteFromTemplate%3Fbiburl=https://api.zotero.org/users/6655/collections/7SR56WJZ/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&amp;format=bibtex&amp;limit=100\"\n      >create a new website with this list</a>, or\n      embed it in an existing web page by copying &amp; pasting\n      any of the following snippets.\n\n      <div style=\"text-align: left; margin-top: 1em;\">\n        <strong>JavaScript</strong>\n        <span class=\"comment recommended\">(easiest)</span>\n        <div class=\"well well-small\">\n          <code>\n            &lt;script src=\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F7SR56WJZ%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1&amp;owner=none&amp;jsonp=1\"&gt;&lt;/script&gt;\n          </code>\n        </div>\n\n        <strong>PHP</strong>\n        <div class=\"well well-small\">\n          <code>\n            &lt;?php\n            $contents = file_get_contents(\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F7SR56WJZ%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1&amp;owner=none\");\n            print_r($contents);\n            ?&gt;\n          </code>\n        </div>\n\n        <strong>iFrame</strong>\n        <span class=\"comment\">(not recommended)</span>\n        <div class=\"well well-small\">\n          <code>\n            &lt;iframe src=\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F7SR56WJZ%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1&amp;owner=none\"&gt;&lt;/iframe&gt;\n          </code>\n        </div>\n\n        <p>\n          For more details see the <a href=\"//bibbase.org/help\">documention</a>.\n        </p>\n      </div>\n    </div>\n\n    <div class=\"alert alert-success bibbase_msg\" id=\"bibbase_msg_preview\"\n         style=\"display: none;\">\n\n      This is a preview! To use this list on your own web site\n      or create a new web site from it,\n      <a href=\"/network/register?next=/network/newAccountWithFile%3FfileId=\"\n      >create a free account</a>. The file will be added\n      and you will be able to edit it in the File Manager.\n      We will show you instructions once you've created your account.\n    </div>\n\n    <div class=\"alert alert-warning bibbase_msg\" id=\"bibbase_msg_mendeley1\"\n         style=\"display: none;\">\n\n      <p>To the site owner:</p>\n\n      <p><strong>Action required!</strong> Mendeley is changing its\n        API. In order to keep using Mendeley with BibBase past April\n        14th, you need to:\n        <ol>\n          <li>renew the authorization for BibBase on Mendeley, and</li>\n          <li>update the BibBase URL\n            in your page the same way you did when you initially set up\n            this page.\n          </li>\n        </ol>\n      </p>\n\n      <p>\n        <a href=\"http://bibbase.org/cgi-bin/mendeley2/get.py\">\n          <i class=\"fa fa-angle-double-right\"></i>\n          Fix it now</a>\n      </p>\n    </div>\n\n  </div>\n\n\n  <div id=\"bibbase_body\">\n    \n  \n  <div class=\"bibbase_group_whole\" id=\"group_2024\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2024')\"\n      onkeypress=\"toggleGroup('group_2024')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2024</span>\n      <span class=\"bibbase_group_count\">\n        \n        (1)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'https://md.ekstrandom.net/pubs/ecir-fair-grids', event);\">\n    Towards optimizing ranking in grid-layout for provider-side fairness.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  In <i>Advances in Information Retrieval</i>, volume 14612, of <i>LNCS</i>, pages 90–105, March 2024. Springer\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"\n     onclick=\"log_download('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'https://md.ekstrandom.net/pubs/ecir-fair-grids', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Towards optimizing ranking in grid-layout for provider-side fairness [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-031-56069-9_7\"\n     onclick=\"log_download('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'http://doi.org/10.1007/978-3-031-56069-9_7', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_towards_2024,\n\tseries = {{LNCS}},\n\ttitle = {Towards optimizing ranking in grid-layout for provider-side fairness},\n\tvolume = {14612},\n\tcopyright = {All rights reserved},\n\turl = {https://md.ekstrandom.net/pubs/ecir-fair-grids},\n\tdoi = {10.1007/978-3-031-56069-9_7},\n\tabstract = {Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.},\n\tlanguage = {en},\n\turldate = {2024-01-04},\n\tbooktitle = {Advances in {Information} {Retrieval}},\n\tpublisher = {Springer},\n\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\n\tmonth = mar,\n\tyear = {2024},\n\tpages = {90--105},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2023\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2023')\"\n      onkeypress=\"toggleGroup('group_2023')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2023</span>\n      <span class=\"bibbase_group_count\">\n        \n        (1)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2309.10271\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023', 'http://arxiv.org/abs/2309.10271', event);\">\n    Towards measuring fairness in grid layout in recommender systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  September 2023.\n  <span class=\"note\">arXiv:2309.10271 [cs]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2309.10271\"\n     onclick=\"log_download('raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023', 'http://arxiv.org/abs/2309.10271', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Towards measuring fairness in grid layout in recommender systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.48550/arXiv.2309.10271\"\n     onclick=\"log_download('raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023', 'http://doi.org/10.48550/arXiv.2309.10271', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-ekstrand-towardsmeasuringfairnessingridlayoutinrecommendersystems-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@misc{raj_towards_2023,\n\ttitle = {Towards measuring fairness in grid layout in recommender systems},\n\tcopyright = {All rights reserved},\n\turl = {http://arxiv.org/abs/2309.10271},\n\tdoi = {10.48550/arXiv.2309.10271},\n\tabstract = {There has been significant research in the last five years on ensuring the providers of items in a recommender system are treated fairly, particularly in terms of the exposure the system provides to their work through its results. However, the metrics developed to date have all been designed and tested for linear ranked lists. It is unknown whether and how existing fair ranking metrics for linear layouts can be applied to grid-based displays. Moreover, depending on the device (phone, tab, or laptop) users use to interact with systems, column size is adjusted using column reduction approaches in a grid-view. The visibility or exposure of recommended items in grid layouts varies based on column sizes and column reduction approaches as well. In this paper, we extend existing fair ranking concepts and metrics to study provider-side group fairness in grid layouts, present an analysis of the behavior of these grid adaptations of fair ranking metrics, and study how their behavior changes across different grid ranking layout designs and geometries. We examine how fairness scores change with different ranking layouts to yield insights into (1) the consistency of fair ranking measurements across layouts; (2) whether rankings optimized for fairness in a linear ranking remain fair when the results are displayed in a grid; and (3) the impact of column reduction approaches to support different device geometries on fairness measurement. This work highlights the need to use layout-specific user attention models when measuring fairness of rankings, and provide practitioners with a first set of insights on what to expect when translating existing fair ranking metrics to the grid layouts in wide use today.},\n\turldate = {2023-11-16},\n\tpublisher = {arXiv},\n\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\n\tmonth = sep,\n\tyear = {2023},\n\tnote = {arXiv:2309.10271 [cs]},\n\tkeywords = {Computer Science - Information Retrieval},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  There has been significant research in the last five years on ensuring the providers of items in a recommender system are treated fairly, particularly in terms of the exposure the system provides to their work through its results. However, the metrics developed to date have all been designed and tested for linear ranked lists. It is unknown whether and how existing fair ranking metrics for linear layouts can be applied to grid-based displays. Moreover, depending on the device (phone, tab, or laptop) users use to interact with systems, column size is adjusted using column reduction approaches in a grid-view. The visibility or exposure of recommended items in grid layouts varies based on column sizes and column reduction approaches as well. In this paper, we extend existing fair ranking concepts and metrics to study provider-side group fairness in grid layouts, present an analysis of the behavior of these grid adaptations of fair ranking metrics, and study how their behavior changes across different grid ranking layout designs and geometries. We examine how fairness scores change with different ranking layouts to yield insights into (1) the consistency of fair ranking measurements across layouts; (2) whether rankings optimized for fairness in a linear ranking remain fair when the results are displayed in a grid; and (3) the impact of column reduction approaches to support different device geometries on fairness measurement. This work highlights the need to use layout-specific user attention models when measuring fairness of rankings, and provide practitioners with a first set of insights on what to expect when translating existing fair ranking metrics to the grid layouts in wide use today.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2022\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2022')\"\n      onkeypress=\"toggleGroup('group_2022')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2022</span>\n      <span class=\"bibbase_group_count\">\n        \n        (1)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/fair-ranking\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'https://md.ekstrandom.net/pubs/fair-ranking', event);\">\n    Measuring fairness in ranked results: an analytical and empirical comparison.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.;  and Ekstrand, M. D\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval</i>, pages 726–736, July 2022. ACM\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/fair-ranking\"\n     onclick=\"log_download('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'https://md.ekstrandom.net/pubs/fair-ranking', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Measuring fairness in ranked results: an analytical and empirical comparison [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3477495.3532018\"\n     onclick=\"log_download('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'http://doi.org/10.1145/3477495.3532018', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>4 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_measuring_2022,\n\ttitle = {Measuring fairness in ranked results: an analytical and empirical comparison},\n\turl = {https://md.ekstrandom.net/pubs/fair-ranking},\n\tdoi = {10.1145/3477495.3532018},\n\tabstract = {Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user&#x27;s information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system&#x27;s behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking.\n\nWe aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.},\n\tbooktitle = {Proceedings of the 45th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\n\tpublisher = {ACM},\n\tauthor = {Raj, Amifa and Ekstrand, Michael D},\n\tmonth = jul,\n\tyear = {2022},\n\tpages = {726--736},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking. We aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2021\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2021')\"\n      onkeypress=\"toggleGroup('group_2021')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2021</span>\n      <span class=\"bibbase_group_count\">\n        \n        (2)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://arxiv.org/abs/2108.05152\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021', 'https://arxiv.org/abs/2108.05152', event);\">\n    Estimation of fair ranking metrics with incomplete judgments.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Kırnap, Ö.; Diaz, F.; Biega, A.; Ekstrand, M.; Carterette, B.;  and Yilmaz, E.\n</span>\n\n  \n\n</span>\n\n  In <i>WWW '21</i>, pages 1065–1075, New York, NY, USA, April 2021. Association for Computing Machinery\n  <span class=\"note\">Journal Abbreviation: WWW '21</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://arxiv.org/abs/2108.05152\"\n     onclick=\"log_download('krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021', 'https://arxiv.org/abs/2108.05152', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Estimation of fair ranking metrics with incomplete judgments [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3442381.3450080\"\n     onclick=\"log_download('krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021', 'http://doi.org/10.1145/3442381.3450080', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('krnap-diaz-biega-ekstrand-carterette-yilmaz-estimationoffairrankingmetricswithincompletejudgments-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{kirnap_estimation_2021,\n\taddress = {New York, NY, USA},\n\ttitle = {Estimation of fair ranking metrics with incomplete judgments},\n\turl = {https://arxiv.org/abs/2108.05152},\n\tdoi = {10.1145/3442381.3450080},\n\tabstract = {There is increasing attention to evaluating the fairness of search system\nranking decisions. These metrics often consider the membership of items to\nparticular groups, often identified using protected attributes such as\ngender or ethnicity. To date, these metrics typically assume the\navailability and completeness of protected attribute labels of items.\nHowever, the protected attributes of individuals are rarely present,\nlimiting the application of fair ranking metrics in large scale systems.\nIn order to address this problem, we propose a sampling strategy and\nestimation technique for four fair ranking metrics. We formulate a robust\nand unbiased estimator which can operate even with very limited number of\nlabeled items. We evaluate our approach using both simulated and real\nworld data. Our experimental results demonstrate that our method can\nestimate this family of fair ranking metrics and provides a robust,\nreliable alternative to exhaustive or random data annotation.},\n\turldate = {2021-04-27},\n\tbooktitle = {{WWW} &#x27;21},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Kırnap, Ömer and Diaz, Fernando and Biega, Asia and Ekstrand, Michael and Carterette, Ben and Yilmaz, Emine},\n\tmonth = apr,\n\tyear = {2021},\n\tnote = {Journal Abbreviation: WWW &#x27;21},\n\tkeywords = {evaluation, information retrieval, fair ranking, fairness},\n\tpages = {1065--1075},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  There is increasing attention to evaluating the fairness of search system ranking decisions. These metrics often consider the membership of items to particular groups, often identified using protected attributes such as gender or ethnicity. To date, these metrics typically assume the availability and completeness of protected attribute labels of items. However, the protected attributes of individuals are rarely present, limiting the application of fair ranking metrics in large scale systems. In order to address this problem, we propose a sampling strategy and estimation technique for four fair ranking metrics. We formulate a robust and unbiased estimator which can operate even with very limited number of labeled items. We evaluate our approach using both simulated and real world data. Our experimental results demonstrate that our method can estimate this family of fair ranking metrics and provides a robust, reliable alternative to exhaustive or random data annotation.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/bag-extended\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'https://md.ekstrandom.net/pubs/bag-extended', event);\">\n    Exploring author gender in book rating and recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ekstrand, M. D;  and Kluver, D.\n</span>\n\n  \n\n</span>\n\n  <i>User Modeling and User-Adapted Interaction</i>, 31(3): 377–420. July 2021.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/bag-extended\"\n     onclick=\"log_download('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'https://md.ekstrandom.net/pubs/bag-extended', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Exploring author gender in book rating and recommendation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s11257-020-09284-2\"\n     onclick=\"log_download('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'http://doi.org/10.1007/s11257-020-09284-2', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>12 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{ekstrand_exploring_2021,\n\ttitle = {Exploring author gender in book rating and recommendation},\n\tvolume = {31},\n\tissn = {0924-1868},\n\turl = {https://md.ekstrandom.net/pubs/bag-extended},\n\tdoi = {10.1007/s11257-020-09284-2},\n\tabstract = {Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.},\n\tnumber = {3},\n\turldate = {2020-06-05},\n\tjournal = {User Modeling and User-Adapted Interaction},\n\tauthor = {Ekstrand, Michael D and Kluver, Daniel},\n\tmonth = jul,\n\tyear = {2021},\n\tpages = {377--420},\n}\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2020\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2020')\"\n      onkeypress=\"toggleGroup('group_2020')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2020</span>\n      <span class=\"bibbase_group_count\">\n        \n        (1)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2009.01311\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020', 'http://arxiv.org/abs/2009.01311', event);\">\n    Comparing fair ranking metrics.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.; Wood, C.; Montoly, A.;  and Ekstrand, M. D\n</span>\n\n  \n\n</span>\n\n  In September 2020. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2009.01311\"\n     onclick=\"log_download('raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020', 'http://arxiv.org/abs/2009.01311', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Comparing fair ranking metrics [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_comparing_2020,\n\ttitle = {Comparing fair ranking metrics},\n\turl = {http://arxiv.org/abs/2009.01311},\n\tabstract = {Ranking is a fundamental aspect of recommender systems. However, ranked\noutputs can be susceptible to various biases; some of these may cause\ndisadvantages to members of protected groups. Several metrics have been\nproposed to quantify the (un)fairness of rankings, but there has not been\nto date any direct comparison of these metrics. This complicates deciding\nwhat fairness metrics are applicable for specific scenarios, and assessing\nthe extent to which metrics agree or disagree. In this paper, we describe\nseveral fair ranking metrics in a common notation, enabling direct\ncomparison of their approaches and assumptions, and empirically compare\nthem on the same experimental setup and data set. Our work provides a\ndirect comparative analysis identifying similarities and differences of\nfair ranking metrics selected for our work.},\n\tauthor = {Raj, Amifa and Wood, Connor and Montoly, Ananda and Ekstrand, Michael D},\n\tmonth = sep,\n\tyear = {2020},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Ranking is a fundamental aspect of recommender systems. However, ranked outputs can be susceptible to various biases; some of these may cause disadvantages to members of protected groups. Several metrics have been proposed to quantify the (un)fairness of rankings, but there has not been to date any direct comparison of these metrics. This complicates deciding what fairness metrics are applicable for specific scenarios, and assessing the extent to which metrics agree or disagree. In this paper, we describe several fair ranking metrics in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data set. Our work provides a direct comparative analysis identifying similarities and differences of fair ranking metrics selected for our work.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2018\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2018')\"\n      onkeypress=\"toggleGroup('group_2018')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2018</span>\n      <span class=\"bibbase_group_count\">\n        \n        (1)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3240323.3240373\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018', 'https://dl.acm.org/doi/10.1145/3240323.3240373', event);\">\n    Exploring author gender in book rating and recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ekstrand, M. D; Tian, M.; Kazi, M. R I.; Mehrpouyan, H.;  and Kluver, D.\n</span>\n\n  \n\n</span>\n\n  In New York, NY, USA, September 2018. ACM\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3240323.3240373\"\n     onclick=\"log_download('ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018', 'https://dl.acm.org/doi/10.1145/3240323.3240373', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Exploring author gender in book rating and recommendation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3240323.3240373\"\n     onclick=\"log_download('ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018', 'http://doi.org/10.1145/3240323.3240373', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ekstrand-tian-kazi-mehrpouyan-kluver-exploringauthorgenderinbookratingandrecommendation-2018')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{ekstrand_exploring_2018,\n\taddress = {New York, NY, USA},\n\ttitle = {Exploring author gender in book rating and recommendation},\n\turl = {https://dl.acm.org/doi/10.1145/3240323.3240373},\n\tdoi = {10.1145/3240323.3240373},\n\tabstract = {Collaborative filtering algorithms find useful patterns in rating and\nconsumption data and exploit these patterns to guide users to good items.\nMany of the patterns in rating datasets reflect important real-world\ndifferences between the various users and items in the data; other\npatterns may be irrelevant or possibly undesirable for social or ethical\nreasons, particularly if they reflect undesired discrimination, such as\ngender or ethnic discrimination in publishing. In this work, we examine\nthe response of collaborative filtering recommender algorithms to the\ndistribution of their input data with respect to a dimension of social\nconcern, namely content creator gender. Using publicly-available book\nratings data, we measure the distribution of the genders of the authors of\nbooks in user rating profiles and recommendation lists produced from this\ndata. We find that common collaborative filtering algorithms differ in the\ngender distribution of their recommendation lists, and in the relationship\nof that output distribution to user profile distribution.},\n\tpublisher = {ACM},\n\tauthor = {Ekstrand, Michael D and Tian, Mucun and Kazi, Mohammed R Imran and Mehrpouyan, Hoda and Kluver, Daniel},\n\tmonth = sep,\n\tyear = {2018},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as gender or ethnic discrimination in publishing. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n\n\n\n  </div>\n\n\n  <!-- Modal -->\n\n  <!-- <iframe id=\"bibbase_network_frame\" -->\n  <!--         src=\"//bibbase.org/network/control\" -->\n  <!--         style=\"display: none;\"> -->\n  <!-- </iframe> -->\n\n</div>\n"}; document.write(bibbase_data.data);