var bibbase_data = {"data":"<img src=\"//bibbase.org/img/ajax-loader.gif\" id=\"spinner\"\n  style=\"display: none;\" alt=\"Loading..\" />\n\n<div id=\"bibbase\">\n\n  <script type=\"text/javascript\">\n    var bibbase = {\n      params: {\"bib\":\"https://api.zotero.org/users/6655/collections/3TB3KT36/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&format=bibtex&limit=100\",\"jsonp\":\"1\",\"host\":\"bibbase.org\"},\n      data: [{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Measuring fairness in ranked results: an analytical and empirical comparison\",\"url\":\"https://md.ekstrandom.net/pubs/fair-ranking\",\"doi\":\"10.1145/3477495.3532018\",\"abstract\":\"Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking. We aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.\",\"booktitle\":\"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval\",\"publisher\":\"ACM\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2022\",\"pages\":\"726–736\",\"bibtex\":\"@inproceedings{raj_measuring_2022,\\n\\ttitle = {Measuring fairness in ranked results: an analytical and empirical comparison},\\n\\turl = {https://md.ekstrandom.net/pubs/fair-ranking},\\n\\tdoi = {10.1145/3477495.3532018},\\n\\tabstract = {Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking.\\n\\nWe aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.},\\n\\tbooktitle = {Proceedings of the 45th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\\n\\tpublisher = {ACM},\\n\\tauthor = {Raj, Amifa and Ekstrand, Michael D},\\n\\tmonth = jul,\\n\\tyear = {2022},\\n\\tpages = {726--736},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Ekstrand, M. D\"],\"key\":\"raj_measuring_2022\",\"id\":\"raj_measuring_2022\",\"bibbaseid\":\"raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/fair-ranking\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":4,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Candidate set sampling for evaluating top-N recommendation\",\"url\":\"http://arxiv.org/abs/2309.11723\",\"doi\":\"10.1109/WI-IAT59888.2023.00018\",\"abstract\":\"The strategy for selecting candidate sets – the set of items that the recommendation system is expected to rank for each user – is an important decision in carrying out an offline top-$N$ recommender system evaluation. The set of candidates is composed of the union of the user's test items and an arbitrary number of non-relevant items that we refer to as decoys. Previous studies have aimed to understand the effect of different candidate set sizes and selection strategies on evaluation. In this paper, we extend this knowledge by studying the specific interaction of candidate set selection strategies with popularity bias, and use simulation to assess whether sampled candidate sets result in metric estimates that are less biased with respect to the true metric values under complete data that is typically unavailable in ordinary experiments.\",\"urldate\":\"2023-11-08\",\"booktitle\":\"Proceedings of the 22nd IEEE/WIC international conference on web intelligence and intelligent agent technology\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ihemelandu\"],\"firstnames\":[\"Ngozi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"October\",\"year\":\"2023\",\"note\":\"arXiv:2309.11723 [cs]\",\"keywords\":\"Computer Science - Information Retrieval\",\"pages\":\"88–94\",\"bibtex\":\"@inproceedings{ihemelandu_candidate_2023,\\n\\ttitle = {Candidate set sampling for evaluating top-{N} recommendation},\\n\\turl = {http://arxiv.org/abs/2309.11723},\\n\\tdoi = {10.1109/WI-IAT59888.2023.00018},\\n\\tabstract = {The strategy for selecting candidate sets -- the set of items that the recommendation system is expected to rank for each user -- is an important decision in carrying out an offline top-\\\\$N\\\\$ recommender system evaluation. The set of candidates is composed of the union of the user's test items and an arbitrary number of non-relevant items that we refer to as decoys. Previous studies have aimed to understand the effect of different candidate set sizes and selection strategies on evaluation. In this paper, we extend this knowledge by studying the specific interaction of candidate set selection strategies with popularity bias, and use simulation to assess whether sampled candidate sets result in metric estimates that are less biased with respect to the true metric values under complete data that is typically unavailable in ordinary experiments.},\\n\\turldate = {2023-11-08},\\n\\tbooktitle = {Proceedings of the 22nd {IEEE}/{WIC} international conference on web intelligence and intelligent agent technology},\\n\\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\\n\\tmonth = oct,\\n\\tyear = {2023},\\n\\tnote = {arXiv:2309.11723 [cs]},\\n\\tkeywords = {Computer Science - Information Retrieval},\\n\\tpages = {88--94},\\n}\\n\\n\",\"author_short\":[\"Ihemelandu, N.\",\"Ekstrand, M. D.\"],\"key\":\"ihemelandu_candidate_2023\",\"id\":\"ihemelandu_candidate_2023\",\"bibbaseid\":\"ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2309.11723\"},\"keyword\":[\"Computer Science - Information Retrieval\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"CIKM '20\",\"title\":\"Evaluating Stochastic Rankings with Expected Exposure\",\"url\":\"http://arxiv.org/abs/2004.13157\",\"doi\":\"10.1145/3340531.3411962\",\"abstract\":\"We introduce the concept of expected exposure as the average attention ranked items receive from users over repeated samples of the same query. Furthermore, we advocate for the adoption of the principle of equal expected exposure: given a fixed information need, no item receive more or less expected exposure compared to any other item of the same relevance grade. We argue that this principle is desirable for many retrieval objectives and scenarios, including topical diversity and fair ranking. Leveraging user models from existing retrieval metrics, we propose a general evaluation methodology based on expected exposure and draw connections to related metrics in information retrieval evaluation. Importantly, this methodology relaxes classic information retrieval assumptions, allowing a system, in response to a query, to produce a distribution over rankings instead of a single fixed ranking. We study the behavior of the expected exposure metric and stochastic rankers across a variety of information access conditions, including ad hoc retrieval and recommendation. We believe that measuring and optimizing expected exposure metrics using randomization opens a new area for retrieval algorithm development and progress.\",\"booktitle\":\"Proceedings of the 29th ACM International Conference on Information and Knowledge Management\",\"publisher\":\"ACM\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Diaz\"],\"firstnames\":[\"Fernando\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mitra\"],\"firstnames\":[\"Bhaskar\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Biega\"],\"firstnames\":[\"Asia\",\"J\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Carterette\"],\"firstnames\":[\"Ben\"],\"suffixes\":[]}],\"month\":\"October\",\"year\":\"2020\",\"bibtex\":\"@inproceedings{diaz_evaluating_2020,\\n\\tseries = {{CIKM} '20},\\n\\ttitle = {Evaluating {Stochastic} {Rankings} with {Expected} {Exposure}},\\n\\turl = {http://arxiv.org/abs/2004.13157},\\n\\tdoi = {10.1145/3340531.3411962},\\n\\tabstract = {We introduce the concept of expected exposure as the average attention\\nranked items receive from users over repeated samples of the same query.\\nFurthermore, we advocate for the adoption of the principle of equal\\nexpected exposure: given a fixed information need, no item receive more or\\nless expected exposure compared to any other item of the same relevance\\ngrade. We argue that this principle is desirable for many retrieval\\nobjectives and scenarios, including topical diversity and fair ranking.\\nLeveraging user models from existing retrieval metrics, we propose a\\ngeneral evaluation methodology based on expected exposure and draw\\nconnections to related metrics in information retrieval evaluation.\\nImportantly, this methodology relaxes classic information retrieval\\nassumptions, allowing a system, in response to a query, to produce a\\ndistribution over rankings instead of a single fixed ranking. We study the\\nbehavior of the expected exposure metric and stochastic rankers across a\\nvariety of information access conditions, including ad hoc retrieval and\\nrecommendation. We believe that measuring and optimizing expected exposure\\nmetrics using randomization opens a new area for retrieval algorithm\\ndevelopment and progress.},\\n\\tbooktitle = {Proceedings of the 29th {ACM} {International} {Conference} on {Information} and {Knowledge} {Management}},\\n\\tpublisher = {ACM},\\n\\tauthor = {Diaz, Fernando and Mitra, Bhaskar and Ekstrand, Michael D and Biega, Asia J and Carterette, Ben},\\n\\tmonth = oct,\\n\\tyear = {2020},\\n}\\n\\n\",\"author_short\":[\"Diaz, F.\",\"Mitra, B.\",\"Ekstrand, M. D\",\"Biega, A. J\",\"Carterette, B.\"],\"key\":\"diaz_evaluating_2020\",\"id\":\"diaz_evaluating_2020\",\"bibbaseid\":\"diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2004.13157\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"LNCS\",\"title\":\"Towards optimizing ranking in grid-layout for provider-side fairness\",\"volume\":\"14612\",\"copyright\":\"All rights reserved\",\"url\":\"https://md.ekstrandom.net/pubs/ecir-fair-grids\",\"doi\":\"10.1007/978-3-031-56069-9_7\",\"abstract\":\"Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.\",\"language\":\"en\",\"urldate\":\"2024-01-04\",\"booktitle\":\"Advances in Information Retrieval\",\"publisher\":\"Springer\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"March\",\"year\":\"2024\",\"pages\":\"90–105\",\"bibtex\":\"@inproceedings{raj_towards_2024,\\n\\tseries = {{LNCS}},\\n\\ttitle = {Towards optimizing ranking in grid-layout for provider-side fairness},\\n\\tvolume = {14612},\\n\\tcopyright = {All rights reserved},\\n\\turl = {https://md.ekstrandom.net/pubs/ecir-fair-grids},\\n\\tdoi = {10.1007/978-3-031-56069-9_7},\\n\\tabstract = {Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.},\\n\\tlanguage = {en},\\n\\turldate = {2024-01-04},\\n\\tbooktitle = {Advances in {Information} {Retrieval}},\\n\\tpublisher = {Springer},\\n\\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\\n\\tmonth = mar,\\n\\tyear = {2024},\\n\\tpages = {90--105},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Ekstrand, M. D.\"],\"key\":\"raj_towards_2024\",\"id\":\"raj_towards_2024\",\"bibbaseid\":\"raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"LNCS\",\"title\":\"Multiple testing for IR and recommendation system experiments\",\"volume\":\"14610\",\"url\":\"https://md.ekstrandom.net/pubs/ecir-mcp\",\"doi\":\"10.1007/978-3-031-56063-7_37\",\"abstract\":\"While there has been significant research on statistical techniques for comparing two information retrieval (IR) systems, many IR experiments test more than two systems. This can lead to inflated false discoveries due to the multiple-comparison problem (MCP). A few IR studies have investigated multiple comparison procedures; these studies mostly use TREC data and control the familywise error rate. In this study, we extend their investigation to include recommendation system evaluation data as well as multiple comparison procedures that controls for False Discovery Rate (FDR).\",\"language\":\"en\",\"urldate\":\"2024-01-04\",\"booktitle\":\"Proceedings of the 46th European Conference on Information Retrieval\",\"publisher\":\"Springer\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ihemelandu\"],\"firstnames\":[\"Ngozi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"March\",\"year\":\"2024\",\"pages\":\"449–457\",\"bibtex\":\"@inproceedings{ihemelandu_multiple_2024,\\n\\tseries = {{LNCS}},\\n\\ttitle = {Multiple testing for {IR} and recommendation system experiments},\\n\\tvolume = {14610},\\n\\turl = {https://md.ekstrandom.net/pubs/ecir-mcp},\\n\\tdoi = {10.1007/978-3-031-56063-7_37},\\n\\tabstract = {While there has been significant research on statistical techniques for comparing two information retrieval (IR) systems, many IR experiments test more than two systems. This can lead to inflated false discoveries due to the multiple-comparison problem (MCP). A few IR studies have investigated multiple comparison procedures; these studies mostly use TREC data and control the familywise error rate. In this study, we extend their investigation to include recommendation system evaluation data as well as multiple comparison procedures that controls for False Discovery Rate (FDR).},\\n\\tlanguage = {en},\\n\\turldate = {2024-01-04},\\n\\tbooktitle = {Proceedings of the 46th {European} {Conference} on {Information} {Retrieval}},\\n\\tpublisher = {Springer},\\n\\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\\n\\tmonth = mar,\\n\\tyear = {2024},\\n\\tpages = {449--457},\\n}\\n\\n\",\"author_short\":[\"Ihemelandu, N.\",\"Ekstrand, M. D.\"],\"key\":\"ihemelandu_multiple_2024\",\"id\":\"ihemelandu_multiple_2024\",\"bibbaseid\":\"ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/ecir-mcp\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Revealing the Hidden Impact of Top-N Metrics on Optimization in Recommender Systems\",\"isbn\":\"978-3-031-56027-9\",\"doi\":\"10.1007/978-3-031-56027-9_9\",\"abstract\":\"The hyperparameters of recommender systems for top-n predictions are typically optimized to enhance the predictive performance of algorithms. Thereby, the optimization algorithm, e.g., grid search or random search, searches for the best hyperparameter configuration according to an optimization-target metric, like nDCG or Precision. In contrast, the optimized algorithm, e.g., Alternating Least Squares Matrix Factorization or Bayesian Personalized Ranking, internally optimizes a different loss function during training, like squared error or cross-entropy. To tackle this discrepancy, recent work focused on generating loss functions better suited for recommender systems. Yet, when evaluating an algorithm using a top-n metric during optimization, another discrepancy between the optimization-target metric and the training loss has so far been ignored. During optimization, the top-n items are selected for computing a top-n metric; ignoring that the top-n items are selected from the recommendations of a model trained with an entirely different loss function. Item recommendations suitable for optimization-target metrics could be outside the top-n recommended items; hiddenly impacting the optimization performance. Therefore, we were motivated to analyze whether the top-n items are optimal for optimization-target top-n metrics. In pursuit of an answer, we exhaustively evaluate the predictive performance of 250 selection strategies besides selecting the top-n. We extensively evaluate each selection strategy over twelve implicit feedback and eight explicit feedback data sets with eleven recommender systems algorithms. Our results show that there exist selection strategies other than top-n that increase predictive performance for various algorithms and recommendation domains. However, the performance of the top $$\\\\sim 43\\\\%$$∼43%of selection strategies is not significantly different. We discuss the impact of our findings on optimization and re-ranking in recommender systems and feasible solutions. The implementation of our study is publicly available.\",\"language\":\"en\",\"booktitle\":\"Advances in Information Retrieval\",\"publisher\":\"Springer Nature Switzerland\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wegmeth\"],\"firstnames\":[\"Lukas\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Vente\"],\"firstnames\":[\"Tobias\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Purucker\"],\"firstnames\":[\"Lennart\"],\"suffixes\":[]}],\"editor\":[{\"propositions\":[],\"lastnames\":[\"Goharian\"],\"firstnames\":[\"Nazli\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Tonellotto\"],\"firstnames\":[\"Nicola\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"He\"],\"firstnames\":[\"Yulan\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Lipani\"],\"firstnames\":[\"Aldo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"McDonald\"],\"firstnames\":[\"Graham\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Macdonald\"],\"firstnames\":[\"Craig\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ounis\"],\"firstnames\":[\"Iadh\"],\"suffixes\":[]}],\"year\":\"2024\",\"pages\":\"140–156\",\"bibtex\":\"@inproceedings{wegmeth_revealing_2024,\\n\\ttitle = {Revealing the {Hidden} {Impact} of {Top}-{N} {Metrics} on {Optimization} in {Recommender} {Systems}},\\n\\tisbn = {978-3-031-56027-9},\\n\\tdoi = {10.1007/978-3-031-56027-9_9},\\n\\tabstract = {The hyperparameters of recommender systems for top-n predictions are typically optimized to enhance the predictive performance of algorithms. Thereby, the optimization algorithm, e.g., grid search or random search, searches for the best hyperparameter configuration according to an optimization-target metric, like nDCG or Precision. In contrast, the optimized algorithm, e.g., Alternating Least Squares Matrix Factorization or Bayesian Personalized Ranking, internally optimizes a different loss function during training, like squared error or cross-entropy. To tackle this discrepancy, recent work focused on generating loss functions better suited for recommender systems. Yet, when evaluating an algorithm using a top-n metric during optimization, another discrepancy between the optimization-target metric and the training loss has so far been ignored. During optimization, the top-n items are selected for computing a top-n metric; ignoring that the top-n items are selected from the recommendations of a model trained with an entirely different loss function. Item recommendations suitable for optimization-target metrics could be outside the top-n recommended items; hiddenly impacting the optimization performance. Therefore, we were motivated to analyze whether the top-n items are optimal for optimization-target top-n metrics. In pursuit of an answer, we exhaustively evaluate the predictive performance of 250 selection strategies besides selecting the top-n. We extensively evaluate each selection strategy over twelve implicit feedback and eight explicit feedback data sets with eleven recommender systems algorithms. Our results show that there exist selection strategies other than top-n that increase predictive performance for various algorithms and recommendation domains. However, the performance of the top \\\\$\\\\${\\\\textbackslash}sim 43{\\\\textbackslash}\\\\%\\\\$\\\\$∼43\\\\%of selection strategies is not significantly different. We discuss the impact of our findings on optimization and re-ranking in recommender systems and feasible solutions. The implementation of our study is publicly available.},\\n\\tlanguage = {en},\\n\\tbooktitle = {Advances in {Information} {Retrieval}},\\n\\tpublisher = {Springer Nature Switzerland},\\n\\tauthor = {Wegmeth, Lukas and Vente, Tobias and Purucker, Lennart},\\n\\teditor = {Goharian, Nazli and Tonellotto, Nicola and He, Yulan and Lipani, Aldo and McDonald, Graham and Macdonald, Craig and Ounis, Iadh},\\n\\tyear = {2024},\\n\\tpages = {140--156},\\n}\\n\\n\",\"author_short\":[\"Wegmeth, L.\",\"Vente, T.\",\"Purucker, L.\"],\"editor_short\":[\"Goharian, N.\",\"Tonellotto, N.\",\"He, Y.\",\"Lipani, A.\",\"McDonald, G.\",\"Macdonald, C.\",\"Ounis, I.\"],\"key\":\"wegmeth_revealing_2024\",\"id\":\"wegmeth_revealing_2024\",\"bibbaseid\":\"wegmeth-vente-purucker-revealingthehiddenimpactoftopnmetricsonoptimizationinrecommendersystems-2024\",\"role\":\"author\",\"urls\":{},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"LNCS\",\"title\":\"An Empirical Analysis of Intervention Strategies’ Effectiveness for Countering Misinformation Amplification by Recommendation Algorithms\",\"volume\":\"14611\",\"isbn\":\"978-3-031-56066-8\",\"doi\":\"10.1007/978-3-031-56066-8_23\",\"abstract\":\"Social network platforms connect people worldwide, facilitating communication, information sharing, and personal/professional networking. They use recommendation algorithms to personalize content and enhance user experiences. However, these algorithms can unintentionally amplify misinformation by prioritizing engagement over accuracy. For instance, recent works suggest that popularity-based and network-based recommendation algorithms contribute the most to misinformation diffusion. In our study, we present an exploration on two Twitter datasets to understand the impact of intervention techniques on combating misinformation amplification initiated by recommendation algorithms. We simulate various scenarios and evaluate the effectiveness of intervention strategies in social sciences such as Virality Circuit Breakers and accuracy nudges. Our findings highlight that these intervention strategies are generally successful when applied on top of collaborative filtering and content-based recommendation algorithms, while having different levels of effectiveness depending on the number of users keen to spread fake news present in the dataset.\",\"language\":\"en\",\"booktitle\":\"Advances in Information Retrieval\",\"publisher\":\"Springer Nature Switzerland\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Pathak\"],\"firstnames\":[\"Royal\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Spezzano\"],\"firstnames\":[\"Francesca\"],\"suffixes\":[]}],\"editor\":[{\"propositions\":[],\"lastnames\":[\"Goharian\"],\"firstnames\":[\"Nazli\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Tonellotto\"],\"firstnames\":[\"Nicola\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"He\"],\"firstnames\":[\"Yulan\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Lipani\"],\"firstnames\":[\"Aldo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"McDonald\"],\"firstnames\":[\"Graham\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Macdonald\"],\"firstnames\":[\"Craig\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ounis\"],\"firstnames\":[\"Iadh\"],\"suffixes\":[]}],\"year\":\"2024\",\"pages\":\"285–301\",\"bibtex\":\"@inproceedings{pathak_empirical_2024,\\n\\tseries = {{LNCS}},\\n\\ttitle = {An {Empirical} {Analysis} of {Intervention} {Strategies}’ {Effectiveness} for {Countering} {Misinformation} {Amplification} by {Recommendation} {Algorithms}},\\n\\tvolume = {14611},\\n\\tisbn = {978-3-031-56066-8},\\n\\tdoi = {10.1007/978-3-031-56066-8_23},\\n\\tabstract = {Social network platforms connect people worldwide, facilitating communication, information sharing, and personal/professional networking. They use recommendation algorithms to personalize content and enhance user experiences. However, these algorithms can unintentionally amplify misinformation by prioritizing engagement over accuracy. For instance, recent works suggest that popularity-based and network-based recommendation algorithms contribute the most to misinformation diffusion. In our study, we present an exploration on two Twitter datasets to understand the impact of intervention techniques on combating misinformation amplification initiated by recommendation algorithms. We simulate various scenarios and evaluate the effectiveness of intervention strategies in social sciences such as Virality Circuit Breakers and accuracy nudges. Our findings highlight that these intervention strategies are generally successful when applied on top of collaborative filtering and content-based recommendation algorithms, while having different levels of effectiveness depending on the number of users keen to spread fake news present in the dataset.},\\n\\tlanguage = {en},\\n\\tbooktitle = {Advances in {Information} {Retrieval}},\\n\\tpublisher = {Springer Nature Switzerland},\\n\\tauthor = {Pathak, Royal and Spezzano, Francesca},\\n\\teditor = {Goharian, Nazli and Tonellotto, Nicola and He, Yulan and Lipani, Aldo and McDonald, Graham and Macdonald, Craig and Ounis, Iadh},\\n\\tyear = {2024},\\n\\tpages = {285--301},\\n}\\n\\n\",\"author_short\":[\"Pathak, R.\",\"Spezzano, F.\"],\"editor_short\":[\"Goharian, N.\",\"Tonellotto, N.\",\"He, Y.\",\"Lipani, A.\",\"McDonald, G.\",\"Macdonald, C.\",\"Ounis, I.\"],\"key\":\"pathak_empirical_2024\",\"id\":\"pathak_empirical_2024\",\"bibbaseid\":\"pathak-spezzano-anempiricalanalysisofinterventionstrategieseffectivenessforcounteringmisinformationamplificationbyrecommendationalgorithms-2024\",\"role\":\"author\",\"urls\":{},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Modeling uncertainty to improve personalized recommendations via Bayesian deep learning\",\"volume\":\"16\",\"issn\":\"2364-4168\",\"url\":\"https://doi.org/10.1007/s41060-020-00241-1\",\"doi\":\"10.1007/s41060-020-00241-1\",\"abstract\":\"Modeling uncertainty has been a major challenge in developing Machine Learning solutions to solve real world problems in various domains. In Recommender Systems, a typical usage of uncertainty is to balance exploration and exploitation, where the uncertainty helps to guide the selection of new options in exploration. Recent advances in combining Bayesian methods with deep learning enable us to express uncertain status in deep learning models. In this paper, we investigate an approach based on Bayesian deep learning to improve personalized recommendations. We first build deep learning architectures to learn useful representation of user and item inputs for predicting their interactions. We then explore multiple embedding components to accommodate different types of user and item inputs. Based on Bayesian deep learning techniques, a key novelty of our approach is to capture the uncertainty associated with the model output and further utilize it to boost exploration in the context of Recommender Systems. We test the proposed approach in both a Collaborative Filtering and a simulated online recommendation setting. Experimental results on publicly available benchmarks demonstrate the benefits of our approach in improving the recommendation performance.\",\"language\":\"en\",\"number\":\"2\",\"urldate\":\"2024-03-17\",\"journal\":\"International Journal of Data Science and Analytics\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wang\"],\"firstnames\":[\"Xin\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kadıoğlu\"],\"firstnames\":[\"Serdar\"],\"suffixes\":[]}],\"month\":\"August\",\"year\":\"2023\",\"pages\":\"191–201\",\"bibtex\":\"@article{wang_modeling_2023,\\n\\ttitle = {Modeling uncertainty to improve personalized recommendations via {Bayesian} deep learning},\\n\\tvolume = {16},\\n\\tissn = {2364-4168},\\n\\turl = {https://doi.org/10.1007/s41060-020-00241-1},\\n\\tdoi = {10.1007/s41060-020-00241-1},\\n\\tabstract = {Modeling uncertainty has been a major challenge in developing Machine Learning solutions to solve real world problems in various domains. In Recommender Systems, a typical usage of uncertainty is to balance exploration and exploitation, where the uncertainty helps to guide the selection of new options in exploration. Recent advances in combining Bayesian methods with deep learning enable us to express uncertain status in deep learning models. In this paper, we investigate an approach based on Bayesian deep learning to improve personalized recommendations. We first build deep learning architectures to learn useful representation of user and item inputs for predicting their interactions. We then explore multiple embedding components to accommodate different types of user and item inputs. Based on Bayesian deep learning techniques, a key novelty of our approach is to capture the uncertainty associated with the model output and further utilize it to boost exploration in the context of Recommender Systems. We test the proposed approach in both a Collaborative Filtering and a simulated online recommendation setting. Experimental results on publicly available benchmarks demonstrate the benefits of our approach in improving the recommendation performance.},\\n\\tlanguage = {en},\\n\\tnumber = {2},\\n\\turldate = {2024-03-17},\\n\\tjournal = {International Journal of Data Science and Analytics},\\n\\tauthor = {Wang, Xin and Kadıoğlu, Serdar},\\n\\tmonth = aug,\\n\\tyear = {2023},\\n\\tpages = {191--201},\\n}\\n\\n\",\"author_short\":[\"Wang, X.\",\"Kadıoğlu, S.\"],\"key\":\"wang_modeling_2023\",\"id\":\"wang_modeling_2023\",\"bibbaseid\":\"wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1007/s41060-020-00241-1\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Recommendations with minimum exposure guarantees: a post-processing framework\",\"volume\":\"236\",\"issn\":\"0957-4174\",\"shorttitle\":\"Recommendations with minimum exposure guarantees\",\"url\":\"https://www.sciencedirect.com/science/article/pii/S0957417423016664\",\"doi\":\"10.1016/j.eswa.2023.121164\",\"abstract\":\"Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantaged groups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.\",\"urldate\":\"2023-09-19\",\"journal\":\"Expert Systems with Applications\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Lopes\"],\"firstnames\":[\"Ramon\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Alves\"],\"firstnames\":[\"Rodrigo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ledent\"],\"firstnames\":[\"Antoine\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Santos\"],\"firstnames\":[\"Rodrygo\",\"L.\",\"T.\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kloft\"],\"firstnames\":[\"Marius\"],\"suffixes\":[]}],\"month\":\"February\",\"year\":\"2024\",\"keywords\":\"Exposure, Fairness, Integer linear programming, Recommender systems, to-read\",\"pages\":\"121164\",\"bibtex\":\"@article{lopes_recommendations_2024,\\n\\ttitle = {Recommendations with minimum exposure guarantees: a post-processing framework},\\n\\tvolume = {236},\\n\\tissn = {0957-4174},\\n\\tshorttitle = {Recommendations with minimum exposure guarantees},\\n\\turl = {https://www.sciencedirect.com/science/article/pii/S0957417423016664},\\n\\tdoi = {10.1016/j.eswa.2023.121164},\\n\\tabstract = {Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantaged groups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.},\\n\\turldate = {2023-09-19},\\n\\tjournal = {Expert Systems with Applications},\\n\\tauthor = {Lopes, Ramon and Alves, Rodrigo and Ledent, Antoine and Santos, Rodrygo L. T. and Kloft, Marius},\\n\\tmonth = feb,\\n\\tyear = {2024},\\n\\tkeywords = {Exposure, Fairness, Integer linear programming, Recommender systems, to-read},\\n\\tpages = {121164},\\n}\\n\\n\",\"author_short\":[\"Lopes, R.\",\"Alves, R.\",\"Ledent, A.\",\"Santos, R. L. T.\",\"Kloft, M.\"],\"key\":\"lopes_recommendations_2024\",\"id\":\"lopes_recommendations_2024\",\"bibbaseid\":\"lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.sciencedirect.com/science/article/pii/S0957417423016664\"},\"keyword\":[\"Exposure\",\"Fairness\",\"Integer linear programming\",\"Recommender systems\",\"to-read\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"The effect of random seeds for data splitting on recommendation accuracy\",\"abstract\":\"The evaluation of recommender system algorithms depends on randomness, e.g., during randomly splitting data into training and testing data. We suspect that failing to account for randomness in this scenario may lead to misrepresenting the predictive accuracy of recommendation algorithms. To understand the community’s view of the importance of randomness, we conducted a paper study on 39 full papers published at the ACM RecSys 2022 conference. We found that the authors of 26 papers used some variation of a holdout split that requires a random seed. However, only five papers explicitly repeated experiments and averaged their results over different random seeds. This potentially problematic research practice motivated us to analyze the effect of data split random seeds on recommendation accuracy. Therefore, we train three common algorithms on nine public data sets with 20 data split random seeds, evaluate them on two ranking metrics with three different ranking cutoff values 𝑘, and compare the results. In the extreme case with 𝑘 = 1, we show that depending on the data split random seed, the accuracy with traditional recommendation algorithms deviates by up to ∼6.3% from the mean accuracy achieved on the data set. Hence, we show that an algorithm may significantly over- or under-perform when maliciously or negligently selecting a random seed for splitting the data. To showcase a mitigation strategy and better research practice, we compare holdout to cross-validation and show that, again, for 𝑘 = 1, the accuracy of algorithms evaluated with cross-validation deviates only up to ∼2.3% from the mean accuracy achieved on the data set. Furthermore, we found that the deviation becomes smaller the higher the value of 𝑘 for both holdout and cross-validation.\",\"language\":\"en\",\"booktitle\":\"Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES 2023)\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wegmeth\"],\"firstnames\":[\"Lukas\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Vente\"],\"firstnames\":[\"Tobias\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Purucker\"],\"firstnames\":[\"Lennart\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Beel\"],\"firstnames\":[\"Joeran\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2023\",\"keywords\":\"to-read\",\"bibtex\":\"@inproceedings{wegmeth_effect_2023,\\n\\ttitle = {The effect of random seeds for data splitting on recommendation accuracy},\\n\\tabstract = {The evaluation of recommender system algorithms depends on randomness, e.g., during randomly splitting data into training and testing data. We suspect that failing to account for randomness in this scenario may lead to misrepresenting the predictive accuracy of recommendation algorithms. To understand the community’s view of the importance of randomness, we conducted a paper study on 39 full papers published at the ACM RecSys 2022 conference. We found that the authors of 26 papers used some variation of a holdout split that requires a random seed. However, only five papers explicitly repeated experiments and averaged their results over different random seeds. This potentially problematic research practice motivated us to analyze the effect of data split random seeds on recommendation accuracy. Therefore, we train three common algorithms on nine public data sets with 20 data split random seeds, evaluate them on two ranking metrics with three different ranking cutoff values 𝑘, and compare the results. In the extreme case with 𝑘 = 1, we show that depending on the data split random seed, the accuracy with traditional recommendation algorithms deviates by up to ∼6.3\\\\% from the mean accuracy achieved on the data set. Hence, we show that an algorithm may significantly over- or under-perform when maliciously or negligently selecting a random seed for splitting the data. To showcase a mitigation strategy and better research practice, we compare holdout to cross-validation and show that, again, for 𝑘 = 1, the accuracy of algorithms evaluated with cross-validation deviates only up to ∼2.3\\\\% from the mean accuracy achieved on the data set. Furthermore, we found that the deviation becomes smaller the higher the value of 𝑘 for both holdout and cross-validation.},\\n\\tlanguage = {en},\\n\\tbooktitle = {Perspectives on the {Evaluation} of {Recommender} {Systems} {Workshop} ({PERSPECTIVES} 2023)},\\n\\tauthor = {Wegmeth, Lukas and Vente, Tobias and Purucker, Lennart and Beel, Joeran},\\n\\tmonth = sep,\\n\\tyear = {2023},\\n\\tkeywords = {to-read},\\n}\\n\\n\",\"author_short\":[\"Wegmeth, L.\",\"Vente, T.\",\"Purucker, L.\",\"Beel, J.\"],\"key\":\"wegmeth_effect_2023\",\"id\":\"wegmeth_effect_2023\",\"bibbaseid\":\"wegmeth-vente-purucker-beel-theeffectofrandomseedsfordatasplittingonrecommendationaccuracy-2023\",\"role\":\"author\",\"urls\":{},\"keyword\":[\"to-read\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"series\":\"RecSys '23\",\"title\":\"Introducing LensKit-Auto, an experimental automated recommender system (AutoRecSys) toolkit\",\"isbn\":\"9798400702419\",\"url\":\"https://dl.acm.org/doi/10.1145/3604915.3610656\",\"doi\":\"10.1145/3604915.3610656\",\"abstract\":\"LensKit is one of the first and most popular Recommender System libraries. While LensKit offers a wide variety of features, it does not include any optimization strategies or guidelines on how to select and tune LensKit algorithms. LensKit developers have to manually include third-party libraries into their experimental setup or implement optimization strategies by hand to optimize hyperparameters. We found that 63.6% (21 out of 33) of papers using LensKit algorithms for their experiments did not select algorithms or tune hyperparameters. Non-optimized models represent poor baselines and produce less meaningful research results. This demo introduces LensKit-Auto. LensKit-Auto automates the entire Recommender System pipeline and enables LensKit developers to automatically select, optimize, and ensemble LensKit algorithms.\",\"urldate\":\"2023-09-18\",\"booktitle\":\"Proceedings of the 17th ACM Conference on Recommender Systems\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Vente\"],\"firstnames\":[\"Tobias\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Beel\"],\"firstnames\":[\"Joeran\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2023\",\"keywords\":\"Algorithm Selection, AutoRecSys, Automated Recommender Systems, CASH, Hyperparameter Optimization, Recommender Systems\",\"pages\":\"1212–1216\",\"bibtex\":\"@inproceedings{vente_introducing_2023,\\n\\taddress = {New York, NY, USA},\\n\\tseries = {{RecSys} '23},\\n\\ttitle = {Introducing {LensKit}-{Auto}, an experimental automated recommender system ({AutoRecSys}) toolkit},\\n\\tisbn = {9798400702419},\\n\\turl = {https://dl.acm.org/doi/10.1145/3604915.3610656},\\n\\tdoi = {10.1145/3604915.3610656},\\n\\tabstract = {LensKit is one of the first and most popular Recommender System libraries. While LensKit offers a wide variety of features, it does not include any optimization strategies or guidelines on how to select and tune LensKit algorithms. LensKit developers have to manually include third-party libraries into their experimental setup or implement optimization strategies by hand to optimize hyperparameters. We found that 63.6\\\\% (21 out of 33) of papers using LensKit algorithms for their experiments did not select algorithms or tune hyperparameters. Non-optimized models represent poor baselines and produce less meaningful research results. This demo introduces LensKit-Auto. LensKit-Auto automates the entire Recommender System pipeline and enables LensKit developers to automatically select, optimize, and ensemble LensKit algorithms.},\\n\\turldate = {2023-09-18},\\n\\tbooktitle = {Proceedings of the 17th {ACM} {Conference} on {Recommender} {Systems}},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Vente, Tobias and Ekstrand, Michael and Beel, Joeran},\\n\\tmonth = sep,\\n\\tyear = {2023},\\n\\tkeywords = {Algorithm Selection, AutoRecSys, Automated Recommender Systems, CASH, Hyperparameter Optimization, Recommender Systems},\\n\\tpages = {1212--1216},\\n}\\n\\n\",\"author_short\":[\"Vente, T.\",\"Ekstrand, M.\",\"Beel, J.\"],\"key\":\"vente_introducing_2023\",\"id\":\"vente_introducing_2023\",\"bibbaseid\":\"vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3604915.3610656\"},\"keyword\":[\"Algorithm Selection\",\"AutoRecSys\",\"Automated Recommender Systems\",\"CASH\",\"Hyperparameter Optimization\",\"Recommender Systems\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"misc\",\"type\":\"misc\",\"title\":\"Mitigating mainstream bias in recommendation via cost-sensitive learning\",\"url\":\"http://arxiv.org/abs/2307.13632\",\"doi\":\"10.1145/3578337.3605134\",\"abstract\":\"Mainstream bias, where some users receive poor recommendations because their preferences are uncommon or simply because they are less active, is an important aspect to consider regarding fairness in recommender systems. Existing methods to mitigate mainstream bias do not explicitly model the importance of these non-mainstream users or, when they do, it is in a way that is not necessarily compatible with the data and recommendation model at hand. In contrast, we use the recommendation utility as a more generic and implicit proxy to quantify mainstreamness, and propose a simple user-weighting approach to incorporate it into the training process while taking the cost of potential recommendation errors into account. We provide extensive experimental results showing that quantifying mainstreamness via utility is better able at identifying non-mainstream users, and that they are indeed better served when training the model in a cost-sensitive way. This is achieved with negligible or no loss in overall recommendation accuracy, meaning that the models learn a better balance across users. In addition, we show that research of this kind, which evaluates recommendation quality at the individual user level, may not be reliable if not using enough interactions when assessing model performance.\",\"urldate\":\"2023-07-29\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Li\"],\"firstnames\":[\"Roger\",\"Zhe\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Urbano\"],\"firstnames\":[\"Julián\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Hanjalic\"],\"firstnames\":[\"Alan\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2023\",\"note\":\"arXiv:2307.13632 [cs]\",\"keywords\":\"Computer Science - Information Retrieval\",\"bibtex\":\"@misc{li_mitigating_2023,\\n\\ttitle = {Mitigating mainstream bias in recommendation via cost-sensitive learning},\\n\\turl = {http://arxiv.org/abs/2307.13632},\\n\\tdoi = {10.1145/3578337.3605134},\\n\\tabstract = {Mainstream bias, where some users receive poor recommendations because their preferences are uncommon or simply because they are less active, is an important aspect to consider regarding fairness in recommender systems. Existing methods to mitigate mainstream bias do not explicitly model the importance of these non-mainstream users or, when they do, it is in a way that is not necessarily compatible with the data and recommendation model at hand. In contrast, we use the recommendation utility as a more generic and implicit proxy to quantify mainstreamness, and propose a simple user-weighting approach to incorporate it into the training process while taking the cost of potential recommendation errors into account. We provide extensive experimental results showing that quantifying mainstreamness via utility is better able at identifying non-mainstream users, and that they are indeed better served when training the model in a cost-sensitive way. This is achieved with negligible or no loss in overall recommendation accuracy, meaning that the models learn a better balance across users. In addition, we show that research of this kind, which evaluates recommendation quality at the individual user level, may not be reliable if not using enough interactions when assessing model performance.},\\n\\turldate = {2023-07-29},\\n\\tauthor = {Li, Roger Zhe and Urbano, Julián and Hanjalic, Alan},\\n\\tmonth = jul,\\n\\tyear = {2023},\\n\\tnote = {arXiv:2307.13632 [cs]},\\n\\tkeywords = {Computer Science - Information Retrieval},\\n}\\n\\n\",\"author_short\":[\"Li, R. Z.\",\"Urbano, J.\",\"Hanjalic, A.\"],\"key\":\"li_mitigating_2023\",\"id\":\"li_mitigating_2023\",\"bibbaseid\":\"li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2307.13632\"},\"keyword\":[\"Computer Science - Information Retrieval\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"series\":\"SIGIR '23\",\"title\":\"Inference at scale: significance testing for large search and recommendation experiments\",\"copyright\":\"All rights reserved\",\"isbn\":\"978-1-4503-9408-6\",\"shorttitle\":\"Inference at scale\",\"url\":\"https://dl.acm.org/doi/10.1145/3539618.3592004\",\"doi\":\"10.1145/3539618.3592004\",\"abstract\":\"A number of information retrieval studies have been done to assess which statistical techniques are appropriate for comparing systems. However, these studies are focused on TREC-style experiments, which typically have fewer than 100 topics. There is no similar line of work for large search and recommendation experiments; such studies typically have thousands of topics or users and much sparser relevance judgements, so it is not clear if recommendations for analyzing traditional TREC experiments apply to these settings. In this paper, we empirically study the behavior of significance tests with large search and recommendation evaluation data. Our results show that the Wilcoxon and Sign tests show significantly higher Type-1 error rates for large sample sizes than the bootstrap, randomization and t-tests, which were more consistent with the expected error rate. While the statistical tests displayed differences in their power for smaller sample sizes, they showed no difference in their power for large sample sizes. We recommend the sign and Wilcoxon tests should not be used to analyze large scale evaluation results. Our result demonstrate that with Top-N recommendation and large search evaluation data, most tests would have a 100% chance of finding statistically significant results. Therefore, the effect size should be used to determine practical or scientific significance.\",\"urldate\":\"2023-07-23\",\"booktitle\":\"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ihemelandu\"],\"firstnames\":[\"Ngozi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2023\",\"keywords\":\"evaluation, statistical inference\",\"pages\":\"2087–2091\",\"bibtex\":\"@inproceedings{ihemelandu_inference_2023,\\n\\taddress = {New York, NY, USA},\\n\\tseries = {{SIGIR} '23},\\n\\ttitle = {Inference at scale: significance testing for large search and recommendation experiments},\\n\\tcopyright = {All rights reserved},\\n\\tisbn = {978-1-4503-9408-6},\\n\\tshorttitle = {Inference at scale},\\n\\turl = {https://dl.acm.org/doi/10.1145/3539618.3592004},\\n\\tdoi = {10.1145/3539618.3592004},\\n\\tabstract = {A number of information retrieval studies have been done to assess which statistical techniques are appropriate for comparing systems. However, these studies are focused on TREC-style experiments, which typically have fewer than 100 topics. There is no similar line of work for large search and recommendation experiments; such studies typically have thousands of topics or users and much sparser relevance judgements, so it is not clear if recommendations for analyzing traditional TREC experiments apply to these settings. In this paper, we empirically study the behavior of significance tests with large search and recommendation evaluation data. Our results show that the Wilcoxon and Sign tests show significantly higher Type-1 error rates for large sample sizes than the bootstrap, randomization and t-tests, which were more consistent with the expected error rate. While the statistical tests displayed differences in their power for smaller sample sizes, they showed no difference in their power for large sample sizes. We recommend the sign and Wilcoxon tests should not be used to analyze large scale evaluation results. Our result demonstrate that with Top-N recommendation and large search evaluation data, most tests would have a 100\\\\% chance of finding statistically significant results. Therefore, the effect size should be used to determine practical or scientific significance.},\\n\\turldate = {2023-07-23},\\n\\tbooktitle = {Proceedings of the 46th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\\n\\tmonth = jul,\\n\\tyear = {2023},\\n\\tkeywords = {evaluation, statistical inference},\\n\\tpages = {2087--2091},\\n}\\n\\n\",\"author_short\":[\"Ihemelandu, N.\",\"Ekstrand, M. D.\"],\"key\":\"ihemelandu_inference_2023\",\"id\":\"ihemelandu_inference_2023\",\"bibbaseid\":\"ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3539618.3592004\"},\"keyword\":[\"evaluation\",\"statistical inference\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Comparing fair ranking metrics\",\"url\":\"http://arxiv.org/abs/2009.01311\",\"abstract\":\"Ranking is a fundamental aspect of recommender systems. However, ranked outputs can be susceptible to various biases; some of these may cause disadvantages to members of protected groups. Several metrics have been proposed to quantify the (un)fairness of rankings, but there has not been to date any direct comparison of these metrics. This complicates deciding what fairness metrics are applicable for specific scenarios, and assessing the extent to which metrics agree or disagree. In this paper, we describe several fair ranking metrics in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data set. Our work provides a direct comparative analysis identifying similarities and differences of fair ranking metrics selected for our work.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Raj\"],\"firstnames\":[\"Amifa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wood\"],\"firstnames\":[\"Connor\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Montoly\"],\"firstnames\":[\"Ananda\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2020\",\"bibtex\":\"@inproceedings{raj_comparing_2020,\\n\\ttitle = {Comparing fair ranking metrics},\\n\\turl = {http://arxiv.org/abs/2009.01311},\\n\\tabstract = {Ranking is a fundamental aspect of recommender systems. However, ranked\\noutputs can be susceptible to various biases; some of these may cause\\ndisadvantages to members of protected groups. Several metrics have been\\nproposed to quantify the (un)fairness of rankings, but there has not been\\nto date any direct comparison of these metrics. This complicates deciding\\nwhat fairness metrics are applicable for specific scenarios, and assessing\\nthe extent to which metrics agree or disagree. In this paper, we describe\\nseveral fair ranking metrics in a common notation, enabling direct\\ncomparison of their approaches and assumptions, and empirically compare\\nthem on the same experimental setup and data set. Our work provides a\\ndirect comparative analysis identifying similarities and differences of\\nfair ranking metrics selected for our work.},\\n\\tauthor = {Raj, Amifa and Wood, Connor and Montoly, Ananda and Ekstrand, Michael D},\\n\\tmonth = sep,\\n\\tyear = {2020},\\n}\\n\\n\",\"author_short\":[\"Raj, A.\",\"Wood, C.\",\"Montoly, A.\",\"Ekstrand, M. D\"],\"key\":\"raj_comparing_2020\",\"id\":\"raj_comparing_2020\",\"bibbaseid\":\"raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2009.01311\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":0,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Exploring author gender in book rating and recommendation\",\"volume\":\"31\",\"issn\":\"0924-1868\",\"url\":\"https://md.ekstrandom.net/pubs/bag-extended\",\"doi\":\"10.1007/s11257-020-09284-2\",\"abstract\":\"Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\",\"number\":\"3\",\"urldate\":\"2020-06-05\",\"journal\":\"User Modeling and User-Adapted Interaction\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kluver\"],\"firstnames\":[\"Daniel\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2021\",\"pages\":\"377–420\",\"bibtex\":\"@article{ekstrand_exploring_2021,\\n\\ttitle = {Exploring author gender in book rating and recommendation},\\n\\tvolume = {31},\\n\\tissn = {0924-1868},\\n\\turl = {https://md.ekstrandom.net/pubs/bag-extended},\\n\\tdoi = {10.1007/s11257-020-09284-2},\\n\\tabstract = {Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.},\\n\\tnumber = {3},\\n\\turldate = {2020-06-05},\\n\\tjournal = {User Modeling and User-Adapted Interaction},\\n\\tauthor = {Ekstrand, Michael D and Kluver, Daniel},\\n\\tmonth = jul,\\n\\tyear = {2021},\\n\\tpages = {377--420},\\n}\\n\\n\",\"author_short\":[\"Ekstrand, M. D\",\"Kluver, D.\"],\"key\":\"ekstrand_exploring_2021\",\"id\":\"ekstrand_exploring_2021\",\"bibbaseid\":\"ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://md.ekstrandom.net/pubs/bag-extended\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":12,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Distributionally-informed recommender system evaluation\",\"copyright\":\"All rights reserved\",\"url\":\"https://dl.acm.org/doi/10.1145/3613455\",\"doi\":\"10.1145/3613455\",\"abstract\":\"Current practice for evaluating recommender systems typically focuses on point estimates of user-oriented effectiveness metrics or business metrics, sometimes combined with additional metrics for considerations such as diversity and novelty. In this paper, we argue for the need for researchers and practitioners to attend more closely to various distributions that arise from a recommender system (or other information access system) and the sources of uncertainty that lead to these distributions. One immediate implication of our argument is that both researchers and practitioners must report and examine more thoroughly the distribution of utility between and within different stakeholder groups. However, distributions of various forms arise in many more aspects of the recommender systems experimental process, and distributional thinking has substantial ramifications for how we design, evaluate, and present recommender systems evaluation and research results. Leveraging and emphasizing distributions in the evaluation of recommender systems is a necessary step to ensure that the systems provide appropriate and equitably-distributed benefit to the people they affect.\",\"urldate\":\"2023-09-07\",\"journal\":\"ACM Transactions on Recommender Systems\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D.\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Carterette\"],\"firstnames\":[\"Ben\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Diaz\"],\"firstnames\":[\"Fernando\"],\"suffixes\":[]}],\"month\":\"August\",\"year\":\"2023\",\"note\":\"Just Accepted\",\"keywords\":\"distributions, evaluation, exposure, statistics\",\"bibtex\":\"@article{ekstrand_distributionally-informed_2023,\\n\\ttitle = {Distributionally-informed recommender system evaluation},\\n\\tcopyright = {All rights reserved},\\n\\turl = {https://dl.acm.org/doi/10.1145/3613455},\\n\\tdoi = {10.1145/3613455},\\n\\tabstract = {Current practice for evaluating recommender systems typically focuses on point estimates of user-oriented effectiveness metrics or business metrics, sometimes combined with additional metrics for considerations such as diversity and novelty. In this paper, we argue for the need for researchers and practitioners to attend more closely to various distributions that arise from a recommender system (or other information access system) and the sources of uncertainty that lead to these distributions. One immediate implication of our argument is that both researchers and practitioners must report and examine more thoroughly the distribution of utility between and within different stakeholder groups. However, distributions of various forms arise in many more aspects of the recommender systems experimental process, and distributional thinking has substantial ramifications for how we design, evaluate, and present recommender systems evaluation and research results. Leveraging and emphasizing distributions in the evaluation of recommender systems is a necessary step to ensure that the systems provide appropriate and equitably-distributed benefit to the people they affect.},\\n\\turldate = {2023-09-07},\\n\\tjournal = {ACM Transactions on Recommender Systems},\\n\\tauthor = {Ekstrand, Michael D. and Carterette, Ben and Diaz, Fernando},\\n\\tmonth = aug,\\n\\tyear = {2023},\\n\\tnote = {Just Accepted},\\n\\tkeywords = {distributions, evaluation, exposure, statistics},\\n}\\n\\n\",\"author_short\":[\"Ekstrand, M. D.\",\"Carterette, B.\",\"Diaz, F.\"],\"key\":\"ekstrand_distributionally-informed_2023\",\"id\":\"ekstrand_distributionally-informed_2023\",\"bibbaseid\":\"ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3613455\"},\"keyword\":[\"distributions\",\"evaluation\",\"exposure\",\"statistics\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"A Framework and Toolkit for Testing the Correctness of Recommendation Algorithms\",\"url\":\"https://doi.org/10.1145/3591109\",\"doi\":\"10.1145/3591109\",\"abstract\":\"Evaluating recommender systems adequately and thoroughly is an important task. Significant efforts are dedicated to proposing metrics, methods and protocols for doing so. However, there has been little discussion in the recommender systems’ literature on the topic of testing. In this work, we adopt and adapt concepts from the software testing domain, e.g., code coverage, metamorphic testing, or property-based testing, to help researchers to detect and correct faults in recommendation algorithms. We propose a test suite that can be used to validate the correctness of a recommendation algorithm, and thus identify and correct issues that can affect the performance and behavior of these algorithms. Our test suite contains both black box and white box tests at every level of abstraction, i.e., system, integration and unit. To facilitate adoption, we release RecPack Tests, an open-source Python package containing template test implementations. We use it to test four popular Python packages for recommender systems: RecPack, PyLensKit, Surprise and Cornac. Despite the high test coverage of each of these packages, we find that we are still able to uncover undocumented functional requirements and even some bugs. This validates our thesis that testing the correctness of recommendation algorithms can complement traditional methods for evaluating recommendation algorithms.\",\"journal\":\"ACM Trans. Recomm. Syst.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Michiels\"],\"firstnames\":[\"Lien\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Verachtert\"],\"firstnames\":[\"Robin\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ferraro\"],\"firstnames\":[\"Andres\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Falk\"],\"firstnames\":[\"Kim\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Goethals\"],\"firstnames\":[\"Bart\"],\"suffixes\":[]}],\"month\":\"April\",\"year\":\"2023\",\"note\":\"Place: New York, NY, USA Publisher: Association for Computing Machinery\",\"bibtex\":\"@article{michiels_framework_2023,\\n\\ttitle = {A {Framework} and {Toolkit} for {Testing} the {Correctness} of {Recommendation} {Algorithms}},\\n\\turl = {https://doi.org/10.1145/3591109},\\n\\tdoi = {10.1145/3591109},\\n\\tabstract = {Evaluating recommender systems adequately and thoroughly is an important\\ntask. Significant efforts are dedicated to proposing metrics, methods and\\nprotocols for doing so. However, there has been little discussion in the\\nrecommender systems’ literature on the topic of testing. In this work, we\\nadopt and adapt concepts from the software testing domain, e.g., code\\ncoverage, metamorphic testing, or property-based testing, to help\\nresearchers to detect and correct faults in recommendation algorithms. We\\npropose a test suite that can be used to validate the correctness of a\\nrecommendation algorithm, and thus identify and correct issues that can\\naffect the performance and behavior of these algorithms. Our test suite\\ncontains both black box and white box tests at every level of abstraction,\\ni.e., system, integration and unit. To facilitate adoption, we release\\nRecPack Tests, an open-source Python package containing template test\\nimplementations. We use it to test four popular Python packages for\\nrecommender systems: RecPack, PyLensKit, Surprise and Cornac. Despite the\\nhigh test coverage of each of these packages, we find that we are still\\nable to uncover undocumented functional requirements and even some bugs.\\nThis validates our thesis that testing the correctness of recommendation\\nalgorithms can complement traditional methods for evaluating\\nrecommendation algorithms.},\\n\\tjournal = {ACM Trans. Recomm. Syst.},\\n\\tauthor = {Michiels, Lien and Verachtert, Robin and Ferraro, Andres and Falk, Kim and Goethals, Bart},\\n\\tmonth = apr,\\n\\tyear = {2023},\\n\\tnote = {Place: New York, NY, USA\\nPublisher: Association for Computing Machinery},\\n}\\n\\n\",\"author_short\":[\"Michiels, L.\",\"Verachtert, R.\",\"Ferraro, A.\",\"Falk, K.\",\"Goethals, B.\"],\"key\":\"michiels_framework_2023\",\"id\":\"michiels_framework_2023\",\"bibbaseid\":\"michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3591109\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":9,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"series\":\"RecSys '23\",\"title\":\"EasyStudy: Framework for Easy Deployment of User Studies on Recommender Systems\",\"isbn\":\"9798400702419\",\"shorttitle\":\"EasyStudy\",\"url\":\"https://dl.acm.org/doi/10.1145/3604915.3610640\",\"doi\":\"10.1145/3604915.3610640\",\"abstract\":\"Improvements in the recommender systems (RS) domain are not possible without a thorough way to evaluate and compare newly proposed approaches. User studies represent a viable alternative to online and offline evaluation schemes, but despite their numerous benefits, they are only rarely used. One of the main reasons behind this fact is that preparing a user study from scratch involves a lot of extra work on top of a simple algorithm proposal. To simplify this task, we propose EasyStudy, a modular framework built on the credo “Make simple things fast and hard things possible”. It features ready-to-use datasets, preference elicitation methods, incrementally tuned baseline algorithms, study flow plugins, and evaluation metrics. As a result, a simple study comparing several RS can be deployed with just a few clicks, while more complex study designs can still benefit from a range of reusable components, such as preference elicitation. Overall, EasyStudy dramatically decreases the gap between the laboriousness of offline evaluation vs. user studies and, therefore, may contribute towards the more reliable and insightful user-centric evaluation of next-generation RS. The project repository is available from https://bit.ly/easy-study-repo.\",\"urldate\":\"2023-09-18\",\"booktitle\":\"Proceedings of the 17th ACM Conference on Recommender Systems\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Dokoupil\"],\"firstnames\":[\"Patrik\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Peska\"],\"firstnames\":[\"Ladislav\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2023\",\"keywords\":\"Recommender systems, evaluation frameworks, user centric, user studies\",\"pages\":\"1196–1199\",\"bibtex\":\"@inproceedings{dokoupil_easystudy_2023,\\n\\taddress = {New York, NY, USA},\\n\\tseries = {{RecSys} '23},\\n\\ttitle = {{EasyStudy}: {Framework} for {Easy} {Deployment} of {User} {Studies} on {Recommender} {Systems}},\\n\\tisbn = {9798400702419},\\n\\tshorttitle = {{EasyStudy}},\\n\\turl = {https://dl.acm.org/doi/10.1145/3604915.3610640},\\n\\tdoi = {10.1145/3604915.3610640},\\n\\tabstract = {Improvements in the recommender systems (RS) domain are not possible without a thorough way to evaluate and compare newly proposed approaches. User studies represent a viable alternative to online and offline evaluation schemes, but despite their numerous benefits, they are only rarely used. One of the main reasons behind this fact is that preparing a user study from scratch involves a lot of extra work on top of a simple algorithm proposal. To simplify this task, we propose EasyStudy, a modular framework built on the credo “Make simple things fast and hard things possible”. It features ready-to-use datasets, preference elicitation methods, incrementally tuned baseline algorithms, study flow plugins, and evaluation metrics. As a result, a simple study comparing several RS can be deployed with just a few clicks, while more complex study designs can still benefit from a range of reusable components, such as preference elicitation. Overall, EasyStudy dramatically decreases the gap between the laboriousness of offline evaluation vs. user studies and, therefore, may contribute towards the more reliable and insightful user-centric evaluation of next-generation RS. The project repository is available from https://bit.ly/easy-study-repo.},\\n\\turldate = {2023-09-18},\\n\\tbooktitle = {Proceedings of the 17th {ACM} {Conference} on {Recommender} {Systems}},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Dokoupil, Patrik and Peska, Ladislav},\\n\\tmonth = sep,\\n\\tyear = {2023},\\n\\tkeywords = {Recommender systems, evaluation frameworks, user centric, user studies},\\n\\tpages = {1196--1199},\\n}\\n\\n\",\"author_short\":[\"Dokoupil, P.\",\"Peska, L.\"],\"key\":\"dokoupil_easystudy_2023\",\"id\":\"dokoupil_easystudy_2023\",\"bibbaseid\":\"dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3604915.3610640\"},\"keyword\":[\"Recommender systems\",\"evaluation frameworks\",\"user centric\",\"user studies\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Understanding the Contribution of Recommendation Algorithms on Misinformation Recommendation and Misinformation Dissemination on Social Networks\",\"issn\":\"1559-1131\",\"url\":\"https://dl.acm.org/doi/10.1145/3616088\",\"doi\":\"10.1145/3616088\",\"abstract\":\"Social networks are a platform for individuals and organizations to connect with each other and inform, advertise, spread ideas, and ultimately influence opinions. These platforms have been known to propel misinformation. We argue that this could be compounded by the recommender algorithms that these platforms use to suggest items potentially of interest to their users, given the known biases and filter bubbles issues affecting recommender systems. While much has been studied about misinformation on social networks, the potential exacerbation that could result from recommender algorithms in this environment is in its infancy. In this manuscript, we present the result of an in-depth analysis conducted on two datasets (Politifact FakeNewsNet dataset and HealthStory FakeHealth dataset) in order to deepen our understanding of the interconnection between recommender algorithms and misinformation spread on Twitter. In particular, we explore the degree to which well-known recommendation algorithms are prone to be impacted by misinformation. Via simulation, we also study misinformation diffusion on social networks, as triggered by suggestions produced by these recommendation algorithms. Outcomes from this work evidence that misinformation does not equally affect all recommendation algorithms. Popularity-based and network-based recommender algorithms contribute the most to misinformation diffusion. Users who are known to be superspreaders are known to directly impact algorithmic performance and misinformation spread in specific scenarios. Findings emerging from our exploration result in a number of implications for researchers and practitioners to consider when designing and deploying recommender algorithms in social networks.\",\"urldate\":\"2023-09-19\",\"journal\":\"ACM Transactions on the Web\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Pathak\"],\"firstnames\":[\"Royal\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Spezzano\"],\"firstnames\":[\"Francesca\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Pera\"],\"firstnames\":[\"Maria\",\"Soledad\"],\"suffixes\":[]}],\"month\":\"August\",\"year\":\"2023\",\"note\":\"Just Accepted\",\"keywords\":\"Twitter, diffusion, misinformation, news, recommendation algorithms, social networks\",\"bibtex\":\"@article{pathak_understanding_2023,\\n\\ttitle = {Understanding the {Contribution} of {Recommendation} {Algorithms} on {Misinformation} {Recommendation} and {Misinformation} {Dissemination} on {Social} {Networks}},\\n\\tissn = {1559-1131},\\n\\turl = {https://dl.acm.org/doi/10.1145/3616088},\\n\\tdoi = {10.1145/3616088},\\n\\tabstract = {Social networks are a platform for individuals and organizations to connect with each other and inform, advertise, spread ideas, and ultimately influence opinions. These platforms have been known to propel misinformation. We argue that this could be compounded by the recommender algorithms that these platforms use to suggest items potentially of interest to their users, given the known biases and filter bubbles issues affecting recommender systems. While much has been studied about misinformation on social networks, the potential exacerbation that could result from recommender algorithms in this environment is in its infancy. In this manuscript, we present the result of an in-depth analysis conducted on two datasets (Politifact FakeNewsNet dataset and HealthStory FakeHealth dataset) in order to deepen our understanding of the interconnection between recommender algorithms and misinformation spread on Twitter. In particular, we explore the degree to which well-known recommendation algorithms are prone to be impacted by misinformation. Via simulation, we also study misinformation diffusion on social networks, as triggered by suggestions produced by these recommendation algorithms. Outcomes from this work evidence that misinformation does not equally affect all recommendation algorithms. Popularity-based and network-based recommender algorithms contribute the most to misinformation diffusion. Users who are known to be superspreaders are known to directly impact algorithmic performance and misinformation spread in specific scenarios. Findings emerging from our exploration result in a number of implications for researchers and practitioners to consider when designing and deploying recommender algorithms in social networks.},\\n\\turldate = {2023-09-19},\\n\\tjournal = {ACM Transactions on the Web},\\n\\tauthor = {Pathak, Royal and Spezzano, Francesca and Pera, Maria Soledad},\\n\\tmonth = aug,\\n\\tyear = {2023},\\n\\tnote = {Just Accepted},\\n\\tkeywords = {Twitter, diffusion, misinformation, news, recommendation algorithms, social networks},\\n}\\n\\n\",\"author_short\":[\"Pathak, R.\",\"Spezzano, F.\",\"Pera, M. S.\"],\"key\":\"pathak_understanding_2023\",\"id\":\"pathak_understanding_2023\",\"bibbaseid\":\"pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3616088\"},\"keyword\":[\"Twitter\",\"diffusion\",\"misinformation\",\"news\",\"recommendation algorithms\",\"social networks\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Measuring the effect of collaborative filtering on the diversity of users’ attention\",\"volume\":\"8\",\"copyright\":\"2023 The Author(s)\",\"issn\":\"2364-8228\",\"url\":\"https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7\",\"doi\":\"10.1007/s41109-022-00530-7\",\"abstract\":\"While the ever-increasing emergence of online services has led to a growing interest in the development of recommender systems, the algorithms underpinning such systems have begun to be criticized for their role in limiting the variety of content exposed to users. In this context, the notion of diversity has been proposed as a way of mitigating the side effects resulting from the specialization of recommender systems. In this paper, using a well-known recommender system that makes use of collaborative filtering in the context of musical content, we analyze the diversity of recommendations generated through the lens of the recently proposed information network diversity measure. The results of our study offer significant insights into the effect of algorithmic recommendations. On the one hand, we show that the musical selections of a large proportion of users are diversified as a result of the recommendations. On the other hand, however, such improvements do not benefit all users. They are in fact mainly restricted to users with a low level of activity or whose past musical listening selections are very narrow. Through more in-depth investigations, we also discovered that while recommendations generally increase the variety of the songs recommended to users, they nonetheless fail to provide a balanced exposure to the different related categories.\",\"language\":\"en\",\"number\":\"1\",\"urldate\":\"2023-09-19\",\"journal\":\"Applied Network Science\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Godinot\"],\"firstnames\":[\"Augustin\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Tarissan\"],\"firstnames\":[\"Fabien\"],\"suffixes\":[]}],\"month\":\"December\",\"year\":\"2023\",\"note\":\"Number: 1 Publisher: SpringerOpen\",\"pages\":\"1–18\",\"bibtex\":\"@article{godinot_measuring_2023,\\n\\ttitle = {Measuring the effect of collaborative filtering on the diversity of users’ attention},\\n\\tvolume = {8},\\n\\tcopyright = {2023 The Author(s)},\\n\\tissn = {2364-8228},\\n\\turl = {https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7},\\n\\tdoi = {10.1007/s41109-022-00530-7},\\n\\tabstract = {While the ever-increasing emergence of online services has led to a growing interest in the development of recommender systems, the algorithms underpinning such systems have begun to be criticized for their role in limiting the variety of content exposed to users. In this context, the notion of diversity has been proposed as a way of mitigating the side effects resulting from the specialization of recommender systems. In this paper, using a well-known recommender system that makes use of collaborative filtering in the context of musical content, we analyze the diversity of recommendations generated through the lens of the recently proposed information network diversity measure. The results of our study offer significant insights into the effect of algorithmic recommendations. On the one hand, we show that the musical selections of a large proportion of users are diversified as a result of the recommendations. On the other hand, however, such improvements do not benefit all users. They are in fact mainly restricted to users with a low level of activity or whose past musical listening selections are very narrow. Through more in-depth investigations, we also discovered that while recommendations generally increase the variety of the songs recommended to users, they nonetheless fail to provide a balanced exposure to the different related categories.},\\n\\tlanguage = {en},\\n\\tnumber = {1},\\n\\turldate = {2023-09-19},\\n\\tjournal = {Applied Network Science},\\n\\tauthor = {Godinot, Augustin and Tarissan, Fabien},\\n\\tmonth = dec,\\n\\tyear = {2023},\\n\\tnote = {Number: 1\\nPublisher: SpringerOpen},\\n\\tpages = {1--18},\\n}\\n\\n\",\"author_short\":[\"Godinot, A.\",\"Tarissan, F.\"],\"key\":\"godinot_measuring_2023\",\"id\":\"godinot_measuring_2023\",\"bibbaseid\":\"godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"series\":\"CHIIR '20\",\"title\":\"Estimating Error and Bias in Offline Evaluation Results\",\"url\":\"https://doi.org/10.1145/3343413.3378004\",\"doi\":\"10.1145/3343413.3378004\",\"urldate\":\"2020-03-26\",\"booktitle\":\"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval\",\"publisher\":\"ACM\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Tian\"],\"firstnames\":[\"Mucun\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ekstrand\"],\"firstnames\":[\"Michael\",\"D\"],\"suffixes\":[]}],\"month\":\"March\",\"year\":\"2020\",\"note\":\"Journal Abbreviation: CHIIR '20\",\"keywords\":\"offline evaluation, simulation\",\"pages\":\"392–396\",\"bibtex\":\"@inproceedings{tian_estimating_2020,\\n\\tseries = {{CHIIR} '20},\\n\\ttitle = {Estimating {Error} and {Bias} in {Offline} {Evaluation} {Results}},\\n\\turl = {https://doi.org/10.1145/3343413.3378004},\\n\\tdoi = {10.1145/3343413.3378004},\\n\\turldate = {2020-03-26},\\n\\tbooktitle = {Proceedings of the 2020 {Conference} on {Human} {Information} {Interaction} and {Retrieval}},\\n\\tpublisher = {ACM},\\n\\tauthor = {Tian, Mucun and Ekstrand, Michael D},\\n\\tmonth = mar,\\n\\tyear = {2020},\\n\\tnote = {Journal Abbreviation: CHIIR '20},\\n\\tkeywords = {offline evaluation, simulation},\\n\\tpages = {392--396},\\n}\\n\\n\",\"author_short\":[\"Tian, M.\",\"Ekstrand, M. D\"],\"key\":\"tian_estimating_2020\",\"id\":\"tian_estimating_2020\",\"bibbaseid\":\"tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3343413.3378004\"},\"keyword\":[\"offline evaluation\",\"simulation\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":4,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"series\":\"UMAP '23 Adjunct\",\"title\":\"The Effect of Similarity Metric and Group Size on Outlier Selection & Satisfaction in Group Recommender Systems\",\"isbn\":\"978-1-4503-9891-6\",\"url\":\"https://dl.acm.org/doi/10.1145/3563359.3597386\",\"doi\":\"10.1145/3563359.3597386\",\"abstract\":\"Group recommender systems (GRS) are a specific case of recommender systems (RS), where recommendations are constructed to a group of users rather than an individual. GRS has diverse application areas including trip planning, recommending movies to watch together, or music in shared environments. However, due to the lack of large datasets with group decision-making feedback information, or even the group definitions, GRS approaches are often evaluated offline w.r.t. individual user feedback and artificially generated groups. These synthetic groups are usually constructed w.r.t. pre-defined group size and inter-user similarity metric. While numerous variants of synthetic group generation procedures were utilized so far, its impact on the evaluation results was not sufficiently discussed. In this paper, we address this research gap by investigating the impact of various synthetic group generation procedures, namely the usage of different user similarity metrics and the effect of group sizes. We consider them in the context of “outlier vs. majority” groups, where a group of similar users is extended with one or more diverse ones. Experimental results indicate a strong impact of the selected similarity metric on both the typical characteristics of selected outliers as well as the performance of individual GRS algorithms. Moreover, we show that certain algorithms better adapt to larger groups than others.\",\"urldate\":\"2023-06-25\",\"booktitle\":\"Adjunct Proceedings of the 31st ACM Conference on User Modeling, Adaptation and Personalization\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Dokoupil\"],\"firstnames\":[\"Patrik\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Peska\"],\"firstnames\":[\"Ladislav\"],\"suffixes\":[]}],\"month\":\"June\",\"year\":\"2023\",\"keywords\":\"Group Recommender systems, Synthetic groups construction, User similarity metrics\",\"pages\":\"296–301\",\"bibtex\":\"@inproceedings{dokoupil_effect_2023,\\n\\taddress = {New York, NY, USA},\\n\\tseries = {{UMAP} '23 {Adjunct}},\\n\\ttitle = {The {Effect} of {Similarity} {Metric} and {Group} {Size} on {Outlier} {Selection} \\\\& {Satisfaction} in {Group} {Recommender} {Systems}},\\n\\tisbn = {978-1-4503-9891-6},\\n\\turl = {https://dl.acm.org/doi/10.1145/3563359.3597386},\\n\\tdoi = {10.1145/3563359.3597386},\\n\\tabstract = {Group recommender systems (GRS) are a specific case of recommender systems (RS), where recommendations are constructed to a group of users rather than an individual. GRS has diverse application areas including trip planning, recommending movies to watch together, or music in shared environments. However, due to the lack of large datasets with group decision-making feedback information, or even the group definitions, GRS approaches are often evaluated offline w.r.t. individual user feedback and artificially generated groups. These synthetic groups are usually constructed w.r.t. pre-defined group size and inter-user similarity metric. While numerous variants of synthetic group generation procedures were utilized so far, its impact on the evaluation results was not sufficiently discussed. In this paper, we address this research gap by investigating the impact of various synthetic group generation procedures, namely the usage of different user similarity metrics and the effect of group sizes. We consider them in the context of “outlier vs. majority” groups, where a group of similar users is extended with one or more diverse ones. Experimental results indicate a strong impact of the selected similarity metric on both the typical characteristics of selected outliers as well as the performance of individual GRS algorithms. Moreover, we show that certain algorithms better adapt to larger groups than others.},\\n\\turldate = {2023-06-25},\\n\\tbooktitle = {Adjunct {Proceedings} of the 31st {ACM} {Conference} on {User} {Modeling}, {Adaptation} and {Personalization}},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Dokoupil, Patrik and Peska, Ladislav},\\n\\tmonth = jun,\\n\\tyear = {2023},\\n\\tkeywords = {Group Recommender systems, Synthetic groups construction, User similarity metrics},\\n\\tpages = {296--301},\\n}\\n\\n\",\"author_short\":[\"Dokoupil, P.\",\"Peska, L.\"],\"key\":\"dokoupil_effect_2023\",\"id\":\"dokoupil_effect_2023\",\"bibbaseid\":\"dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://dl.acm.org/doi/10.1145/3563359.3597386\"},\"keyword\":[\"Group Recommender systems\",\"Synthetic groups construction\",\"User similarity metrics\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"title\":\"FairRecKit: A Web-based Analysis Software for Recommender Evaluations\",\"url\":\"https://doi.org/10.1145/3576840.3578274\",\"doi\":\"10.1145/3576840.3578274\",\"abstract\":\"FairRecKit is a web-based analysis software that supports researchers in performing, analyzing, and understanding recommendation computations. The idea behind FairRecKit is to facilitate the in-depth analysis of recommendation outcomes considering fairness aspects. With (nested) filters on user or item attributes, metrics can easily be compared across user and item subgroups. Further, (nested) filters can be used on the dataset level; this way, recommendation outcomes can be compared across several sub-datasets to analyze for differences considering fairness aspects. The software currently features five datasets, 11 metrics, and 21 recommendation algorithms to be used in computational experimentation. It is open source and developed in a modular manner to facilitate extension. The analysis software consists of two components: A software package (FairRecKitLib) for running recommendation algorithms on the available datasets and a web-based user interface (FairRecKitApp) to start experiments, retrieve results of previous experiments, and analyze details. The application also comes with extensive documentation and options for result customization, which makes for a flexible tool that supports in-depth analysis.\",\"urldate\":\"2023-03-28\",\"booktitle\":\"CHIIR '23\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Bauer\"],\"firstnames\":[\"Christine\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Chung\"],\"firstnames\":[\"Lennard\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Cornelissen\"],\"firstnames\":[\"Aleksej\"],\"suffixes\":[]},{\"propositions\":[\"van\"],\"lastnames\":[\"Driessel\"],\"firstnames\":[\"Isabelle\"],\"suffixes\":[]},{\"propositions\":[\"van\",\"der\"],\"lastnames\":[\"Hoorn\"],\"firstnames\":[\"Diede\"],\"suffixes\":[]},{\"propositions\":[\"de\"],\"lastnames\":[\"Jong\"],\"firstnames\":[\"Yme\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Le\"],\"firstnames\":[\"Lan\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Najiyan\",\"Tabriz\"],\"firstnames\":[\"Sanaz\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Spaans\"],\"firstnames\":[\"Roderick\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Thijsen\"],\"firstnames\":[\"Casper\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Verbeeten\"],\"firstnames\":[\"Robert\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wesseling\"],\"firstnames\":[\"Vos\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wieland\"],\"firstnames\":[\"Fern\"],\"suffixes\":[]}],\"month\":\"March\",\"year\":\"2023\",\"note\":\"Journal Abbreviation: CHIIR '23\",\"keywords\":\"evaluation, music, web-based, recommender systems, toolkit, analysis, movies, resource\",\"pages\":\"438–443\",\"bibtex\":\"@inproceedings{bauer_fairreckit_2023,\\n\\taddress = {New York, NY, USA},\\n\\ttitle = {{FairRecKit}: {A} {Web}-based {Analysis} {Software} for {Recommender} {Evaluations}},\\n\\turl = {https://doi.org/10.1145/3576840.3578274},\\n\\tdoi = {10.1145/3576840.3578274},\\n\\tabstract = {FairRecKit is a web-based analysis software that supports researchers in\\nperforming, analyzing, and understanding recommendation computations. The\\nidea behind FairRecKit is to facilitate the in-depth analysis of\\nrecommendation outcomes considering fairness aspects. With (nested)\\nfilters on user or item attributes, metrics can easily be compared across\\nuser and item subgroups. Further, (nested) filters can be used on the\\ndataset level; this way, recommendation outcomes can be compared across\\nseveral sub-datasets to analyze for differences considering fairness\\naspects. The software currently features five datasets, 11 metrics, and 21\\nrecommendation algorithms to be used in computational experimentation. It\\nis open source and developed in a modular manner to facilitate extension.\\nThe analysis software consists of two components: A software package\\n(FairRecKitLib) for running recommendation algorithms on the available\\ndatasets and a web-based user interface (FairRecKitApp) to start\\nexperiments, retrieve results of previous experiments, and analyze\\ndetails. The application also comes with extensive documentation and\\noptions for result customization, which makes for a flexible tool that\\nsupports in-depth analysis.},\\n\\turldate = {2023-03-28},\\n\\tbooktitle = {{CHIIR} '23},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Bauer, Christine and Chung, Lennard and Cornelissen, Aleksej and van Driessel, Isabelle and van der Hoorn, Diede and de Jong, Yme and Le, Lan and Najiyan Tabriz, Sanaz and Spaans, Roderick and Thijsen, Casper and Verbeeten, Robert and Wesseling, Vos and Wieland, Fern},\\n\\tmonth = mar,\\n\\tyear = {2023},\\n\\tnote = {Journal Abbreviation: CHIIR '23},\\n\\tkeywords = {evaluation, music, web-based, recommender systems, toolkit, analysis, movies, resource},\\n\\tpages = {438--443},\\n}\\n\\n\",\"author_short\":[\"Bauer, C.\",\"Chung, L.\",\"Cornelissen, A.\",\"van Driessel, I.\",\"van der Hoorn, D.\",\"de Jong, Y.\",\"Le, L.\",\"Najiyan Tabriz, S.\",\"Spaans, R.\",\"Thijsen, C.\",\"Verbeeten, R.\",\"Wesseling, V.\",\"Wieland, F.\"],\"key\":\"bauer_fairreckit_2023\",\"id\":\"bauer_fairreckit_2023\",\"bibbaseid\":\"bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3576840.3578274\"},\"keyword\":[\"evaluation\",\"music\",\"web-based\",\"recommender systems\",\"toolkit\",\"analysis\",\"movies\",\"resource\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":5,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"A Comparative Study of Rank Aggregation Methods in Recommendation Systems\",\"volume\":\"25\",\"issn\":\"1099-4300\",\"url\":\"https://www.mdpi.com/1099-4300/25/1/132\",\"doi\":\"10.3390/e25010132\",\"abstract\":\"The aim of a recommender system is to suggest to the user certain products or services that most likely will interest them. Within the context of personalized recommender systems, a number of algorithms have been suggested to generate a ranking of items tailored to individual user preferences. However, these algorithms do not generate identical recommendations, and for this reason it has been suggested in the literature that the results of these algorithms can be combined using aggregation techniques, hoping that this will translate into an improvement in the quality of the final recommendation. In order to see which of these techniques increase the quality of recommendations to the greatest extent, the authors of this publication conducted experiments in which they considered five recommendation algorithms and 20 aggregation methods. The research was carried out on the popular and publicly available MovieLens 100k and MovieLens 1M datasets, and the results were confirmed by statistical tests.\",\"number\":\"1\",\"urldate\":\"2023-01-12\",\"journal\":\"Entropy\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Bałchanowski\"],\"firstnames\":[\"Michał\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Boryczka\"],\"firstnames\":[\"Urszula\"],\"suffixes\":[]}],\"month\":\"January\",\"year\":\"2023\",\"note\":\"Publisher: Multidisciplinary Digital Publishing Institute\",\"pages\":\"132\",\"bibtex\":\"@article{balchanowski_comparative_2023,\\n\\ttitle = {A {Comparative} {Study} of {Rank} {Aggregation} {Methods} in {Recommendation} {Systems}},\\n\\tvolume = {25},\\n\\tissn = {1099-4300},\\n\\turl = {https://www.mdpi.com/1099-4300/25/1/132},\\n\\tdoi = {10.3390/e25010132},\\n\\tabstract = {The aim of a recommender system is to suggest to the user certain products\\nor services that most likely will interest them. Within the context of\\npersonalized recommender systems, a number of algorithms have been\\nsuggested to generate a ranking of items tailored to individual user\\npreferences. However, these algorithms do not generate identical\\nrecommendations, and for this reason it has been suggested in the\\nliterature that the results of these algorithms can be combined using\\naggregation techniques, hoping that this will translate into an\\nimprovement in the quality of the final recommendation. In order to see\\nwhich of these techniques increase the quality of recommendations to the\\ngreatest extent, the authors of this publication conducted experiments in\\nwhich they considered five recommendation algorithms and 20 aggregation\\nmethods. The research was carried out on the popular and publicly\\navailable MovieLens 100k and MovieLens 1M datasets, and the results were\\nconfirmed by statistical tests.},\\n\\tnumber = {1},\\n\\turldate = {2023-01-12},\\n\\tjournal = {Entropy},\\n\\tauthor = {Bałchanowski, Michał and Boryczka, Urszula},\\n\\tmonth = jan,\\n\\tyear = {2023},\\n\\tnote = {Publisher: Multidisciplinary Digital Publishing Institute},\\n\\tpages = {132},\\n}\\n\\n\",\"author_short\":[\"Bałchanowski, M.\",\"Boryczka, U.\"],\"key\":\"balchanowski_comparative_2023\",\"id\":\"balchanowski_comparative_2023\",\"bibbaseid\":\"bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.mdpi.com/1099-4300/25/1/132\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"What You Like: Generating Explainable Topical Recommendations for Twitter Using Social Annotations\",\"url\":\"http://arxiv.org/abs/2212.13897\",\"abstract\":\"With over 500 million tweets posted per day, in Twitter, it is difficult for Twitter users to discover interesting content from the deluge of uninteresting posts. In this work, we present a novel, explainable, topical recommendation system, that utilizes social annotations, to help Twitter users discover tweets, on topics of their interest. A major challenge in using traditional rating dependent recommendation systems, like collaborative filtering and content based systems, in high volume social networks is that, due to attention scarcity most items do not get any ratings. Additionally, the fact that most Twitter users are passive consumers, with 44% users never tweeting, makes it very difficult to use user ratings for generating recommendations. Further, a key challenge in developing recommendation systems is that in many cases users reject relevant recommendations if they are totally unfamiliar with the recommended item. Providing a suitable explanation, for why the item is recommended, significantly improves the acceptability of recommendation. By virtue of being a topical recommendation system our method is able to present simple topical explanations for the generated recommendations. Comparisons with state-of-the-art matrix factorization based collaborative filtering, content based and social recommendations demonstrate the efficacy of the proposed approach.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Bhattacharya\"],\"firstnames\":[\"Parantapa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ghosh\"],\"firstnames\":[\"Saptarshi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Zafar\"],\"firstnames\":[\"Muhammad\",\"Bilal\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ghosh\"],\"firstnames\":[\"Soumya\",\"K\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ganguly\"],\"firstnames\":[\"Niloy\"],\"suffixes\":[]}],\"month\":\"December\",\"year\":\"2022\",\"note\":\"ISBN: 2212.13897 Publication Title: arXiv [cs.IR]\",\"bibtex\":\"@unpublished{bhattacharya_what_2022,\\n\\ttitle = {What {You} {Like}: {Generating} {Explainable} {Topical} {Recommendations} for {Twitter} {Using} {Social} {Annotations}},\\n\\turl = {http://arxiv.org/abs/2212.13897},\\n\\tabstract = {With over 500 million tweets posted per day, in Twitter, it is difficult\\nfor Twitter users to discover interesting content from the deluge of\\nuninteresting posts. In this work, we present a novel, explainable,\\ntopical recommendation system, that utilizes social annotations, to help\\nTwitter users discover tweets, on topics of their interest. A major\\nchallenge in using traditional rating dependent recommendation systems,\\nlike collaborative filtering and content based systems, in high volume\\nsocial networks is that, due to attention scarcity most items do not get\\nany ratings. Additionally, the fact that most Twitter users are passive\\nconsumers, with 44\\\\% users never tweeting, makes it very difficult to use\\nuser ratings for generating recommendations. Further, a key challenge in\\ndeveloping recommendation systems is that in many cases users reject\\nrelevant recommendations if they are totally unfamiliar with the\\nrecommended item. Providing a suitable explanation, for why the item is\\nrecommended, significantly improves the acceptability of recommendation.\\nBy virtue of being a topical recommendation system our method is able to\\npresent simple topical explanations for the generated recommendations.\\nComparisons with state-of-the-art matrix factorization based collaborative\\nfiltering, content based and social recommendations demonstrate the\\nefficacy of the proposed approach.},\\n\\tauthor = {Bhattacharya, Parantapa and Ghosh, Saptarshi and Zafar, Muhammad Bilal and Ghosh, Soumya K and Ganguly, Niloy},\\n\\tmonth = dec,\\n\\tyear = {2022},\\n\\tnote = {ISBN: 2212.13897\\nPublication Title: arXiv [cs.IR]},\\n}\\n\\n\",\"author_short\":[\"Bhattacharya, P.\",\"Ghosh, S.\",\"Zafar, M. B.\",\"Ghosh, S. K\",\"Ganguly, N.\"],\"key\":\"bhattacharya_what_2022\",\"id\":\"bhattacharya_what_2022\",\"bibbaseid\":\"bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2212.13897\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"Recommendations with Minimum Exposure Guarantees: A Post-Processing Framework\",\"url\":\"https://papers.ssrn.com/abstract=4274780\",\"abstract\":\"Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantagedgroups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.\",\"urldate\":\"2022-12-23\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Lopes\"],\"firstnames\":[\"Ramon\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Alves\"],\"firstnames\":[\"Rodrigo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ledent\"],\"firstnames\":[\"Antoine\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Santos\"],\"firstnames\":[\"Rodrygo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kloft\"],\"firstnames\":[\"Marius\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2022\",\"doi\":\"10.2139/ssrn.4274780\",\"keywords\":\"recommender systems, fairness, exposure, integer linear programming\",\"bibtex\":\"@unpublished{lopes_recommendations_2022,\\n\\ttitle = {Recommendations with {Minimum} {Exposure} {Guarantees}: {A} {Post}-{Processing} {Framework}},\\n\\turl = {https://papers.ssrn.com/abstract=4274780},\\n\\tabstract = {Relevance-based ranking is a popular ingredient in recommenders, but it\\nfrequently struggles to meet fairness criteria because social and cultural\\nnorms may favor some item groups over others. For instance, some items\\nmight receive lower ratings due to some sort of bias (e.g. gender bias). A\\nfair ranking should balance the exposure of items from advantaged and\\ndisadvantagedgroups. To this end, we propose a novel post-processing\\nframework to produce fair, exposure-aware recommendations. Our approach is\\nbased on an integer linear programming model maximizing the expected\\nutility while satisfying a minimum exposure constraint. The model has\\nfewer variables than previous work and thus can be deployed to larger\\ndatasets and allows the organization to define a minimum level of exposure\\nfor groups of items. We conduct an extensive empirical evaluation\\nindicating that our new framework can increase the exposure of items from\\ndisadvantaged groups at a small cost of recommendation accuracy.},\\n\\turldate = {2022-12-23},\\n\\tauthor = {Lopes, Ramon and Alves, Rodrigo and Ledent, Antoine and Santos, Rodrygo and Kloft, Marius},\\n\\tmonth = nov,\\n\\tyear = {2022},\\n\\tdoi = {10.2139/ssrn.4274780},\\n\\tkeywords = {recommender systems, fairness, exposure, integer linear programming},\\n}\\n\\n\",\"author_short\":[\"Lopes, R.\",\"Alves, R.\",\"Ledent, A.\",\"Santos, R.\",\"Kloft, M.\"],\"key\":\"lopes_recommendations_2022\",\"id\":\"lopes_recommendations_2022\",\"bibbaseid\":\"lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://papers.ssrn.com/abstract=4274780\"},\"keyword\":[\"recommender systems\",\"fairness\",\"exposure\",\"integer linear programming\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Collaborative Rank Aggregation in Recommendation Systems\",\"volume\":\"207\",\"issn\":\"1877-0509\",\"url\":\"https://www.sciencedirect.com/science/article/pii/S187705092201167X\",\"doi\":\"10.1016/j.procs.2022.09.281\",\"abstract\":\"Over the years, various techniques of generating recommendations have been developed. However, it turns out that when we compare the recommendations generated by different algorithms in the context of a particular user, the quality of such recommendations for different techniques may differ. The use of the aggregation techniques, the aim of which is to combine several rankings into one, can be a solution to this problem. In theory it should improve the quality of the recommendations. Additionally, in order to personalize the recommendations better, a metaheuristic algorithm, which, by assigning different weights to each feature, tries to represent the preference of the active user, was used. This paper also presents a suggestion to include additional rankings generated for other users in the system in the aggregation process. The idea will be supported by research results that clearly show that taking into account rankings of other users can improve the quality of the generated recommendations.\",\"journal\":\"Procedia Comput. Sci.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Bałchanowski\"],\"firstnames\":[\"Michał\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Boryczka\"],\"firstnames\":[\"Urszula\"],\"suffixes\":[]}],\"month\":\"January\",\"year\":\"2022\",\"keywords\":\"differential evolution, metaheuristic, rank aggregation, recommendation systems\",\"pages\":\"2213–2222\",\"bibtex\":\"@article{balchanowski_collaborative_2022,\\n\\ttitle = {Collaborative {Rank} {Aggregation} in {Recommendation} {Systems}},\\n\\tvolume = {207},\\n\\tissn = {1877-0509},\\n\\turl = {https://www.sciencedirect.com/science/article/pii/S187705092201167X},\\n\\tdoi = {10.1016/j.procs.2022.09.281},\\n\\tabstract = {Over the years, various techniques of generating recommendations have been\\ndeveloped. However, it turns out that when we compare the recommendations\\ngenerated by different algorithms in the context of a particular user, the\\nquality of such recommendations for different techniques may differ. The\\nuse of the aggregation techniques, the aim of which is to combine several\\nrankings into one, can be a solution to this problem. In theory it should\\nimprove the quality of the recommendations. Additionally, in order to\\npersonalize the recommendations better, a metaheuristic algorithm, which,\\nby assigning different weights to each feature, tries to represent the\\npreference of the active user, was used. This paper also presents a\\nsuggestion to include additional rankings generated for other users in the\\nsystem in the aggregation process. The idea will be supported by research\\nresults that clearly show that taking into account rankings of other users\\ncan improve the quality of the generated recommendations.},\\n\\tjournal = {Procedia Comput. Sci.},\\n\\tauthor = {Bałchanowski, Michał and Boryczka, Urszula},\\n\\tmonth = jan,\\n\\tyear = {2022},\\n\\tkeywords = {differential evolution, metaheuristic, rank aggregation, recommendation systems},\\n\\tpages = {2213--2222},\\n}\\n\\n\",\"author_short\":[\"Bałchanowski, M.\",\"Boryczka, U.\"],\"key\":\"balchanowski_collaborative_2022\",\"id\":\"balchanowski_collaborative_2022\",\"bibbaseid\":\"bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.sciencedirect.com/science/article/pii/S187705092201167X\"},\"keyword\":[\"differential evolution\",\"metaheuristic\",\"rank aggregation\",\"recommendation systems\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"CaMeLS: Cooperative meta-learning service for recommender systems\",\"url\":\"https://ceur-ws.org/Vol-3228/paper2.pdf\",\"abstract\":\"We present CaMeLS, a proof of concept of a cooperative meta-learning service for recommender systems. CaMeLS leverages the computing power of recommender systems users by uploading their metadata and algorithm evaluation scores to a centralized environment. Through the resulting database, CaMeLS then offers meta-learning services for everyone. Additionally, users may access evaluations of common data sets immediately to know the best-performing algorithms for those data sets. The metadata table may also be used for other purposes, e.g., to perform benchmarks. In the initial version discussed in this paper, CaMeLS implements automatic algorithm selection through meta-learning over two recommender systems libraries. Automatic algorithm selection saves users time and computing power and does not require expertise, as the best algorithm is automatically found over multiple libraries. The CaMeLS database contains 20 metadata sets by default. We show that the automatic algorithm selection service is already on par with the single best algorithm in this default scenario. CaMeLS only requires a few seconds to predict a suitable algorithm, rather than potentially hours or days if performed manually, depending on the data set. The code is publicly available on our GitHub https://camels.recommender-systems.com.\",\"urldate\":\"2022-11-13\",\"publisher\":\"CEUR-WS\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wegmeth\"],\"firstnames\":[\"Lukas\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Beel\"],\"firstnames\":[\"Joeran\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2022\",\"bibtex\":\"@inproceedings{wegmeth_camels_2022,\\n\\ttitle = {{CaMeLS}: {Cooperative} meta-learning service for recommender systems},\\n\\turl = {https://ceur-ws.org/Vol-3228/paper2.pdf},\\n\\tabstract = {We present CaMeLS, a proof of concept of a cooperative meta-learning\\nservice for recommender systems. CaMeLS leverages the computing power of\\nrecommender systems users by uploading their metadata and algorithm\\nevaluation scores to a centralized environment. Through the resulting\\ndatabase, CaMeLS then offers meta-learning services for everyone.\\nAdditionally, users may access evaluations of common data sets immediately\\nto know the best-performing algorithms for those data sets. The metadata\\ntable may also be used for other purposes, e.g., to perform benchmarks. In\\nthe initial version discussed in this paper, CaMeLS implements automatic\\nalgorithm selection through meta-learning over two recommender systems\\nlibraries. Automatic algorithm selection saves users time and computing\\npower and does not require expertise, as the best algorithm is\\nautomatically found over multiple libraries. The CaMeLS database contains\\n20 metadata sets by default. We show that the automatic algorithm\\nselection service is already on par with the single best algorithm in this\\ndefault scenario. CaMeLS only requires a few seconds to predict a suitable\\nalgorithm, rather than potentially hours or days if performed manually,\\ndepending on the data set. The code is publicly available on our GitHub\\nhttps://camels.recommender-systems.com.},\\n\\turldate = {2022-11-13},\\n\\tpublisher = {CEUR-WS},\\n\\tauthor = {Wegmeth, Lukas and Beel, Joeran},\\n\\tmonth = sep,\\n\\tyear = {2022},\\n}\\n\\n\",\"author_short\":[\"Wegmeth, L.\",\"Beel, J.\"],\"key\":\"wegmeth_camels_2022\",\"id\":\"wegmeth_camels_2022\",\"bibbaseid\":\"wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://ceur-ws.org/Vol-3228/paper2.pdf\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"Representation with Incomplete Votes\",\"url\":\"http://arxiv.org/abs/2211.15608\",\"abstract\":\"Platforms for online civic participation rely heavily on methods for condensing thousands of comments into a relevant handful, based on whether participants agree or disagree with them. These methods should guarantee fair representation of the participants, as their outcomes may affect the health of the conversation and inform impactful downstream decisions. To that end, we draw on the literature on approval-based committee elections. Our setting is novel in that the approval votes are incomplete since participants will typically not vote on all comments. We prove that this complication renders non-adaptive algorithms impractical in terms of the amount of information they must gather. Therefore, we develop an adaptive algorithm that uses information more efficiently by presenting incoming participants with statements that appear promising based on votes by previous participants. We prove that this method satisfies commonly used notions of fair representation, even when participants only vote on a small fraction of comments. Finally, an empirical evaluation using real data shows that the proposed algorithm provides representative outcomes in practice.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Halpern\"],\"firstnames\":[\"Daniel\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kehne\"],\"firstnames\":[\"Gregory\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Procaccia\"],\"firstnames\":[\"Ariel\",\"D\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Tucker-Foltz\"],\"firstnames\":[\"Jamie\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wüthrich\"],\"firstnames\":[\"Manuel\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2022\",\"note\":\"ISBN: 2211.15608 Publication Title: arXiv [cs.GT]\",\"bibtex\":\"@unpublished{halpern_representation_2022,\\n\\ttitle = {Representation with {Incomplete} {Votes}},\\n\\turl = {http://arxiv.org/abs/2211.15608},\\n\\tabstract = {Platforms for online civic participation rely heavily on methods for\\ncondensing thousands of comments into a relevant handful, based on whether\\nparticipants agree or disagree with them. These methods should guarantee\\nfair representation of the participants, as their outcomes may affect the\\nhealth of the conversation and inform impactful downstream decisions. To\\nthat end, we draw on the literature on approval-based committee elections.\\nOur setting is novel in that the approval votes are incomplete since\\nparticipants will typically not vote on all comments. We prove that this\\ncomplication renders non-adaptive algorithms impractical in terms of the\\namount of information they must gather. Therefore, we develop an adaptive\\nalgorithm that uses information more efficiently by presenting incoming\\nparticipants with statements that appear promising based on votes by\\nprevious participants. We prove that this method satisfies commonly used\\nnotions of fair representation, even when participants only vote on a\\nsmall fraction of comments. Finally, an empirical evaluation using real\\ndata shows that the proposed algorithm provides representative outcomes in\\npractice.},\\n\\tauthor = {Halpern, Daniel and Kehne, Gregory and Procaccia, Ariel D and Tucker-Foltz, Jamie and Wüthrich, Manuel},\\n\\tmonth = nov,\\n\\tyear = {2022},\\n\\tnote = {ISBN: 2211.15608\\nPublication Title: arXiv [cs.GT]},\\n}\\n\\n\",\"author_short\":[\"Halpern, D.\",\"Kehne, G.\",\"Procaccia, A. D\",\"Tucker-Foltz, J.\",\"Wüthrich, M.\"],\"key\":\"halpern_representation_2022\",\"id\":\"halpern_representation_2022\",\"bibbaseid\":\"halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2211.15608\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"User Perception of Recommendation Explanation: Are Your Explanations What Users Need?\",\"issn\":\"1094-9224\",\"url\":\"https://doi.org/10.1145/3565480\",\"doi\":\"10.1145/3565480\",\"abstract\":\"As recommender systems become increasingly important in daily human decision-making, users are demanding convincing explanations to understand they get the specific recommendation results. Although a number of explainable recommender systems have recently been proposed, there still lacks an understanding of what users really need in a recommendation explanation. The actual reason behind users’ intention to examine and consume (e.g., click and watch a movie) can be the window to answer this question and is named as self-explanation in this work. In addition, humans usually make recommendations accompanied by explanations, but there remain fewer studies on how humans explain and what we can learn from human-generated explanations. To investigate these questions, we conduct a novel multi-role, multi-session user study in which users interact with multiple types of system-generated explanations as well as human-generated explanations, namely peer-explanation. During the study, users’ intentions, expectations, and experiences are tracked in several phases, including before and after the users are presented with an explanation and after the content is examined. Through comprehensive investigations, three main findings have been made: First, we observe not only the positive but also the negative effects of explanations, and the impact varies across different types of explanations. Moreover, human-generated explanation, peer-explanation, performs better in increasing user intentions and helping users to better construct preferences, which results in better user satisfaction. Second, based on users’ self-explanation, the information accuracy is measured and found to be a major factor associated with user satisfaction. Some other factors, such as unfamiliarity and similarity, are also discovered and summarized. Third, through annotations of the information aspects used in the human-generated self-explanation and peer-explanation, patterns of how humans explain are investigated, including what information and how much information is utilized. In addition, based on the findings, a human-inspired explanation approach is proposed and found to increase user satisfaction, revealing the potential improvement of further incorporating more human patterns in recommendation explanations. These findings have shed light on the deeper understanding of the recommendation explanation and further research on its evaluation and generation. Furthermore, the collected data, including human-generated explanations by both the external peers and the users’ selves, will be released to support future research works on explanation evaluation.\",\"journal\":\"ACM Trans. Inf. Syst. Secur.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Lu\"],\"firstnames\":[\"Hongyu\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ma\"],\"firstnames\":[\"Weizhi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wang\"],\"firstnames\":[\"Yifan\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Zhang\"],\"firstnames\":[\"Min\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Wang\"],\"firstnames\":[\"Xiang\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Liu\"],\"firstnames\":[\"Yiqun\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Chua\"],\"firstnames\":[\"Tat-Seng\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ma\"],\"firstnames\":[\"Shaoping\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2022\",\"note\":\"Place: New York, NY, USA Publisher: Association for Computing Machinery\",\"keywords\":\"Recommendation Explanation, User Modeling, Recommender System\",\"bibtex\":\"@article{lu_user_2022,\\n\\ttitle = {User {Perception} of {Recommendation} {Explanation}: {Are} {Your} {Explanations} {What} {Users} {Need}?},\\n\\tissn = {1094-9224},\\n\\turl = {https://doi.org/10.1145/3565480},\\n\\tdoi = {10.1145/3565480},\\n\\tabstract = {As recommender systems become increasingly important in daily human\\ndecision-making, users are demanding convincing explanations to understand\\nthey get the specific recommendation results. Although a number of\\nexplainable recommender systems have recently been proposed, there still\\nlacks an understanding of what users really need in a recommendation\\nexplanation. The actual reason behind users’ intention to examine and\\nconsume (e.g., click and watch a movie) can be the window to answer this\\nquestion and is named as self-explanation in this work. In addition,\\nhumans usually make recommendations accompanied by explanations, but there\\nremain fewer studies on how humans explain and what we can learn from\\nhuman-generated explanations. To investigate these questions, we conduct a\\nnovel multi-role, multi-session user study in which users interact with\\nmultiple types of system-generated explanations as well as human-generated\\nexplanations, namely peer-explanation. During the study, users’\\nintentions, expectations, and experiences are tracked in several phases,\\nincluding before and after the users are presented with an explanation and\\nafter the content is examined. Through comprehensive investigations, three\\nmain findings have been made: First, we observe not only the positive but\\nalso the negative effects of explanations, and the impact varies across\\ndifferent types of explanations. Moreover, human-generated explanation,\\npeer-explanation, performs better in increasing user intentions and\\nhelping users to better construct preferences, which results in better\\nuser satisfaction. Second, based on users’ self-explanation, the\\ninformation accuracy is measured and found to be a major factor associated\\nwith user satisfaction. Some other factors, such as unfamiliarity and\\nsimilarity, are also discovered and summarized. Third, through annotations\\nof the information aspects used in the human-generated self-explanation\\nand peer-explanation, patterns of how humans explain are investigated,\\nincluding what information and how much information is utilized. In\\naddition, based on the findings, a human-inspired explanation approach is\\nproposed and found to increase user satisfaction, revealing the potential\\nimprovement of further incorporating more human patterns in recommendation\\nexplanations. These findings have shed light on the deeper understanding\\nof the recommendation explanation and further research on its evaluation\\nand generation. Furthermore, the collected data, including human-generated\\nexplanations by both the external peers and the users’ selves, will be\\nreleased to support future research works on explanation evaluation.},\\n\\tjournal = {ACM Trans. Inf. Syst. Secur.},\\n\\tauthor = {Lu, Hongyu and Ma, Weizhi and Wang, Yifan and Zhang, Min and Wang, Xiang and Liu, Yiqun and Chua, Tat-Seng and Ma, Shaoping},\\n\\tmonth = nov,\\n\\tyear = {2022},\\n\\tnote = {Place: New York, NY, USA\\nPublisher: Association for Computing Machinery},\\n\\tkeywords = {Recommendation Explanation, User Modeling, Recommender System},\\n}\\n\\n\",\"author_short\":[\"Lu, H.\",\"Ma, W.\",\"Wang, Y.\",\"Zhang, M.\",\"Wang, X.\",\"Liu, Y.\",\"Chua, T.\",\"Ma, S.\"],\"key\":\"lu_user_2022\",\"id\":\"lu_user_2022\",\"bibbaseid\":\"lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3565480\"},\"keyword\":[\"Recommendation Explanation\",\"User Modeling\",\"Recommender System\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":3,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"A New Collaborative Filtering Algorithm Integrating Time and Multisimilarity\",\"volume\":\"2022\",\"issn\":\"1024-123X\",\"url\":\"https://www.hindawi.com/journals/mpe/2022/2340671/\",\"doi\":\"10.1155/2022/2340671\",\"abstract\":\"Aiming at the problem of low recommendation accuracy of existing recommendation algorithms, an algorithm integrating time factors and multisimilarity is proposed to improve the impact of long-term data, user attention, and project popularity on the recommendation algorithm and the similarity of user attributes is introduced to improve the problem of cold start to a certain extent. Considering that the longer the time, the less likely it is to be selected again, time is introduced into the algorithm as a weight factor. When the behavior occurs, i.e., interest in the project, so as to judge the similarity between users, not just the score value, we normalize the popularity to avoid misjudgment of high scoring and popular items. Because new users do not have past score records, the problem of cold start can be solved by calculating the similarity of user attributes. Through the comparative experiment on Movielens100K dataset and Epinions dataset, the results show that the algorithm can improve the accuracy of recommendation and give users a better recommendation effect.\",\"urldate\":\"2022-09-05\",\"journal\":\"Math. Probl. Eng.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Liu\"],\"firstnames\":[\"Qin\"],\"suffixes\":[]}],\"month\":\"August\",\"year\":\"2022\",\"note\":\"Publisher: Hindawi\",\"bibtex\":\"@article{liu_new_2022,\\n\\ttitle = {A {New} {Collaborative} {Filtering} {Algorithm} {Integrating} {Time} and {Multisimilarity}},\\n\\tvolume = {2022},\\n\\tissn = {1024-123X},\\n\\turl = {https://www.hindawi.com/journals/mpe/2022/2340671/},\\n\\tdoi = {10.1155/2022/2340671},\\n\\tabstract = {Aiming at the problem of low recommendation accuracy of existing\\nrecommendation algorithms, an algorithm integrating time factors and\\nmultisimilarity is proposed to improve the impact of long-term data, user\\nattention, and project popularity on the recommendation algorithm and the\\nsimilarity of user attributes is introduced to improve the problem of cold\\nstart to a certain extent. Considering that the longer the time, the less\\nlikely it is to be selected again, time is introduced into the algorithm\\nas a weight factor. When the behavior occurs, i.e., interest in the\\nproject, so as to judge the similarity between users, not just the score\\nvalue, we normalize the popularity to avoid misjudgment of high scoring\\nand popular items. Because new users do not have past score records, the\\nproblem of cold start can be solved by calculating the similarity of user\\nattributes. Through the comparative experiment on Movielens100K dataset\\nand Epinions dataset, the results show that the algorithm can improve the\\naccuracy of recommendation and give users a better recommendation effect.},\\n\\turldate = {2022-09-05},\\n\\tjournal = {Math. Probl. Eng.},\\n\\tauthor = {Liu, Qin},\\n\\tmonth = aug,\\n\\tyear = {2022},\\n\\tnote = {Publisher: Hindawi},\\n}\\n\\n\",\"author_short\":[\"Liu, Q.\"],\"key\":\"liu_new_2022\",\"id\":\"liu_new_2022\",\"bibbaseid\":\"liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.hindawi.com/journals/mpe/2022/2340671/\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":3,\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"The Impact of Feature Quantity on Recommendation Algorithm Performance: A Movielens-100K Case Study\",\"url\":\"http://arxiv.org/abs/2207.08713\",\"abstract\":\"Recent model-based Recommender Systems (RecSys) algorithms emphasize on the use of features, also called side information, in their design similar to algorithms in Machine Learning (ML). In contrast, some of the most popular and traditional algorithms for RecSys solely focus on a given user-item-rating relation without including side information. The goal of this case study is to provide a performance comparison and assessment of RecSys and ML algorithms when side information is included. We chose the Movielens-100K data set since it is a standard for comparing RecSys algorithms. We compared six different feature sets with varying quantities of features which were generated from the baseline data and evaluated on a total of 19 RecSys algorithms, baseline ML algorithms, Automated Machine Learning (AutoML) pipelines, and state-of-the-art RecSys algorithms that incorporate side information. The results show that additional features benefit all algorithms we evaluated. However, the correlation between feature quantity and performance is not monotonous for AutoML and RecSys. In these categories, an analysis of feature importance revealed that the quality of features matters more than quantity. Throughout our experiments, the average performance on the feature set with the lowest number of features is about 6% worse compared to that with the highest in terms of the Root Mean Squared Error. An interesting observation is that AutoML outperforms matrix factorization-based RecSys algorithms when additional features are used. Almost all algorithms that can include side information have higher performance when using the highest quantity of features. In the other cases, the performance difference is negligible (\\\\textless1%). The results show a clear positive trend for the effect of feature quantity as well as the important effects of feature quality on the evaluated algorithms.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wegmeth\"],\"firstnames\":[\"Lukas\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2022\",\"note\":\"ISBN: 2207.08713 Publication Title: arXiv [cs.IR]\",\"bibtex\":\"@unpublished{wegmeth_impact_2022,\\n\\ttitle = {The {Impact} of {Feature} {Quantity} on {Recommendation} {Algorithm} {Performance}: {A} {Movielens}-{100K} {Case} {Study}},\\n\\turl = {http://arxiv.org/abs/2207.08713},\\n\\tabstract = {Recent model-based Recommender Systems (RecSys) algorithms emphasize on\\nthe use of features, also called side information, in their design similar\\nto algorithms in Machine Learning (ML). In contrast, some of the most\\npopular and traditional algorithms for RecSys solely focus on a given\\nuser-item-rating relation without including side information. The goal of\\nthis case study is to provide a performance comparison and assessment of\\nRecSys and ML algorithms when side information is included. We chose the\\nMovielens-100K data set since it is a standard for comparing RecSys\\nalgorithms. We compared six different feature sets with varying quantities\\nof features which were generated from the baseline data and evaluated on a\\ntotal of 19 RecSys algorithms, baseline ML algorithms, Automated Machine\\nLearning (AutoML) pipelines, and state-of-the-art RecSys algorithms that\\nincorporate side information. The results show that additional features\\nbenefit all algorithms we evaluated. However, the correlation between\\nfeature quantity and performance is not monotonous for AutoML and RecSys.\\nIn these categories, an analysis of feature importance revealed that the\\nquality of features matters more than quantity. Throughout our\\nexperiments, the average performance on the feature set with the lowest\\nnumber of features is about 6\\\\% worse compared to that with the highest in\\nterms of the Root Mean Squared Error. An interesting observation is that\\nAutoML outperforms matrix factorization-based RecSys algorithms when\\nadditional features are used. Almost all algorithms that can include side\\ninformation have higher performance when using the highest quantity of\\nfeatures. In the other cases, the performance difference is negligible\\n({\\\\textless}1\\\\%). The results show a clear positive trend for the effect of feature\\nquantity as well as the important effects of feature quality on the\\nevaluated algorithms.},\\n\\tauthor = {Wegmeth, Lukas},\\n\\tmonth = jul,\\n\\tyear = {2022},\\n\\tnote = {ISBN: 2207.08713\\nPublication Title: arXiv [cs.IR]},\\n}\\n\\n\",\"author_short\":[\"Wegmeth, L.\"],\"key\":\"wegmeth_impact_2022\",\"id\":\"wegmeth_impact_2022\",\"bibbaseid\":\"wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2207.08713\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":3,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Exploring Set-Inspired Similarity Measures for Collaborative Filtering Recommendation\",\"url\":\"https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Quang-Hung\"],\"firstnames\":[\"L\",\"E\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Thi-Xinh\"],\"firstnames\":[\"L\",\"E\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2021\",\"bibtex\":\"@inproceedings{quang-hung_exploring_2021,\\n\\ttitle = {Exploring {Set}-{Inspired} {Similarity} {Measures} for {Collaborative} {Filtering} {Recommendation}},\\n\\turl = {https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf},\\n\\tauthor = {Quang-Hung, L E and Thi-Xinh, L E},\\n\\tmonth = nov,\\n\\tyear = {2021},\\n}\\n\\n\",\"author_short\":[\"Quang-Hung, L E\",\"Thi-Xinh, L E\"],\"key\":\"quang-hung_exploring_2021\",\"id\":\"quang-hung_exploring_2021\",\"bibbaseid\":\"quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Towards user-oriented privacy for recommender system data: A personalization-based approach to gender obfuscation for user profiles\",\"volume\":\"58\",\"issn\":\"0306-4573\",\"url\":\"https://www.sciencedirect.com/science/article/pii/S0306457321002065\",\"doi\":\"10.1016/j.ipm.2021.102722\",\"abstract\":\"In this paper, we propose a new privacy solution for the data used to train a recommender system, i.e., the user–item matrix. The user–item matrix contains implicit information, which can be inferred using a classifier, leading to potential privacy violations. Our solution, called Personalized Blurring (PerBlur), is a simple, yet effective, approach to adding and removing items from users’ profiles in order to generate an obfuscated user–item matrix. The novelty of PerBlur is personalization of the choice of items used for obfuscation to the individual user profiles. PerBlur is formulated within a user-oriented paradigm of recommender system data privacy that aims at making privacy solutions understandable, unobtrusive, and useful for the user. When obfuscated data is used for training, a recommender system algorithm is able to reach performance comparable to what is attained when it is trained on the original, unobfuscated data. At the same time, a classifier can no longer reliably use the obfuscated data to predict the gender of users, indicating that implicit gender information has been removed. In addition to introducing PerBlur, we make several key contributions. First, we propose an evaluation protocol that creates a fair environment to compare between different obfuscation conditions. Second, we carry out experiments that show that gender obfuscation impacts the fairness and diversity of recommender system results. In sum, our work establishes that a simple, transparent approach to gender obfuscation can protect user privacy while at the same time improving recommendation results for users by maintaining fairness and enhancing diversity.\",\"number\":\"6\",\"journal\":\"Inf. Process. Manag.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Slokom\"],\"firstnames\":[\"Manel\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Hanjalic\"],\"firstnames\":[\"Alan\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Larson\"],\"firstnames\":[\"Martha\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2021\",\"keywords\":\"Diversity, Evaluation, Fairness, Gender inference, Obfuscation, Privacy, Top-N recommendation\",\"pages\":\"102722\",\"bibtex\":\"@article{slokom_towards_2021,\\n\\ttitle = {Towards user-oriented privacy for recommender system data: {A} personalization-based approach to gender obfuscation for user profiles},\\n\\tvolume = {58},\\n\\tissn = {0306-4573},\\n\\turl = {https://www.sciencedirect.com/science/article/pii/S0306457321002065},\\n\\tdoi = {10.1016/j.ipm.2021.102722},\\n\\tabstract = {In this paper, we propose a new privacy solution for the data used to\\ntrain a recommender system, i.e., the user–item matrix. The user–item\\nmatrix contains implicit information, which can be inferred using a\\nclassifier, leading to potential privacy violations. Our solution, called\\nPersonalized Blurring (PerBlur), is a simple, yet effective, approach to\\nadding and removing items from users’ profiles in order to generate an\\nobfuscated user–item matrix. The novelty of PerBlur is personalization of\\nthe choice of items used for obfuscation to the individual user profiles.\\nPerBlur is formulated within a user-oriented paradigm of recommender\\nsystem data privacy that aims at making privacy solutions understandable,\\nunobtrusive, and useful for the user. When obfuscated data is used for\\ntraining, a recommender system algorithm is able to reach performance\\ncomparable to what is attained when it is trained on the original,\\nunobfuscated data. At the same time, a classifier can no longer reliably\\nuse the obfuscated data to predict the gender of users, indicating that\\nimplicit gender information has been removed. In addition to introducing\\nPerBlur, we make several key contributions. First, we propose an\\nevaluation protocol that creates a fair environment to compare between\\ndifferent obfuscation conditions. Second, we carry out experiments that\\nshow that gender obfuscation impacts the fairness and diversity of\\nrecommender system results. In sum, our work establishes that a simple,\\ntransparent approach to gender obfuscation can protect user privacy while\\nat the same time improving recommendation results for users by maintaining\\nfairness and enhancing diversity.},\\n\\tnumber = {6},\\n\\tjournal = {Inf. Process. Manag.},\\n\\tauthor = {Slokom, Manel and Hanjalic, Alan and Larson, Martha},\\n\\tmonth = nov,\\n\\tyear = {2021},\\n\\tkeywords = {Diversity, Evaluation, Fairness, Gender inference, Obfuscation, Privacy, Top-N recommendation},\\n\\tpages = {102722},\\n}\\n\\n\",\"author_short\":[\"Slokom, M.\",\"Hanjalic, A.\",\"Larson, M.\"],\"key\":\"slokom_towards_2021\",\"id\":\"slokom_towards_2021\",\"bibbaseid\":\"slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.sciencedirect.com/science/article/pii/S0306457321002065\"},\"keyword\":[\"Diversity\",\"Evaluation\",\"Fairness\",\"Gender inference\",\"Obfuscation\",\"Privacy\",\"Top-N recommendation\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"Application of Knowledge Graphs to Provide Side Information for Improved Recommendation Accuracy\",\"url\":\"http://arxiv.org/abs/2101.03054\",\"abstract\":\"Personalized recommendations are popular in these days of Internet driven activities, specifically shopping. Recommendation methods can be grouped into three major categories, content based filtering, collaborative filtering and machine learning enhanced. Information about products and preferences of different users are primarily used to infer preferences for a specific user. Inadequate information can obviously cause these methods to fail or perform poorly. The more information we provide to these methods, the more likely it is that the methods perform better. Knowledge graphs represent the current trend in recording information in the form of relations between entities, and can provide additional (side) information about products and users. Such information can be used to improve nearest neighbour search, clustering users and products, or train the neural network, when one is used. In this work, we present a new generic recommendation systems framework, that integrates knowledge graphs into the recommendation pipeline. We describe its software design and implementation, and then show through experiments, how such a framework can be specialized for a domain, say movie recommendations, and the improvements in recommendation results possible due to side information obtained from knowledge graphs representation of such information. Our framework supports different knowledge graph representation formats, and facilitates format conversion, merging and information extraction needed for training recommendation methods.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Mao\"],\"firstnames\":[\"Yuhao\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mokhov\"],\"firstnames\":[\"Serguei\",\"A\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mudur\"],\"firstnames\":[\"Sudhir\",\"P\"],\"suffixes\":[]}],\"month\":\"January\",\"year\":\"2021\",\"doi\":\"10.1007/s11257-018-9213-x\",\"note\":\"ISBN: 2101.03054 Publication Title: arXiv [cs.IR]\",\"bibtex\":\"@unpublished{mao_application_2021,\\n\\ttitle = {Application of {Knowledge} {Graphs} to {Provide} {Side} {Information} for {Improved} {Recommendation} {Accuracy}},\\n\\turl = {http://arxiv.org/abs/2101.03054},\\n\\tabstract = {Personalized recommendations are popular in these days of Internet driven\\nactivities, specifically shopping. Recommendation methods can be grouped\\ninto three major categories, content based filtering, collaborative\\nfiltering and machine learning enhanced. Information about products and\\npreferences of different users are primarily used to infer preferences for\\na specific user. Inadequate information can obviously cause these methods\\nto fail or perform poorly. The more information we provide to these\\nmethods, the more likely it is that the methods perform better. Knowledge\\ngraphs represent the current trend in recording information in the form of\\nrelations between entities, and can provide additional (side) information\\nabout products and users. Such information can be used to improve nearest\\nneighbour search, clustering users and products, or train the neural\\nnetwork, when one is used. In this work, we present a new generic\\nrecommendation systems framework, that integrates knowledge graphs into\\nthe recommendation pipeline. We describe its software design and\\nimplementation, and then show through experiments, how such a framework\\ncan be specialized for a domain, say movie recommendations, and the\\nimprovements in recommendation results possible due to side information\\nobtained from knowledge graphs representation of such information. Our\\nframework supports different knowledge graph representation formats, and\\nfacilitates format conversion, merging and information extraction needed\\nfor training recommendation methods.},\\n\\tauthor = {Mao, Yuhao and Mokhov, Serguei A and Mudur, Sudhir P},\\n\\tmonth = jan,\\n\\tyear = {2021},\\n\\tdoi = {10.1007/s11257-018-9213-x},\\n\\tnote = {ISBN: 2101.03054\\nPublication Title: arXiv [cs.IR]},\\n}\\n\\n\",\"author_short\":[\"Mao, Y.\",\"Mokhov, S. A\",\"Mudur, S. P\"],\"key\":\"mao_application_2021\",\"id\":\"mao_application_2021\",\"bibbaseid\":\"mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2101.03054\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"phdthesis\",\"type\":\"phdthesis\",\"title\":\"Enhancing the Multi-Objective Recommendation from three new perspectives: data characterization, risk-sensitiveness, and prioritization of the objectives\",\"url\":\"https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf\",\"school\":\"Universidade Federal de Minas Gerais\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Fortes\"],\"firstnames\":[\"Renaldo\",\"Silva\"],\"suffixes\":[]}],\"collaborator\":\"Gonçalves, Marcos André\",\"year\":\"2022\",\"note\":\"Publication Title: Ciência da Computação Volume: Doctoral\",\"bibtex\":\"@phdthesis{fortes_enhancing_2022,\\n\\ttitle = {Enhancing the {Multi}-{Objective} {Recommendation} from three new perspectives: data characterization, risk-sensitiveness, and prioritization of the objectives},\\n\\turl = {https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf},\\n\\tschool = {Universidade Federal de Minas Gerais},\\n\\tauthor = {Fortes, Renaldo Silva},\\n\\tcollaborator = {Gonçalves, Marcos André},\\n\\tyear = {2022},\\n\\tnote = {Publication Title: Ciência da Computação\\nVolume: Doctoral},\\n}\\n\\n\",\"author_short\":[\"Fortes, R. S.\"],\"key\":\"fortes_enhancing_2022\",\"id\":\"fortes_enhancing_2022\",\"bibbaseid\":\"fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":5,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Time-dependent evaluation of recommender systems\",\"volume\":\"2955\",\"url\":\"https://perspectives-ws.github.io/2021/proceedings/paper10.pdf\",\"urldate\":\"2021-09-27\",\"booktitle\":\"CEUR-WS\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Scheidt\"],\"firstnames\":[\"Teresa\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Beel\"],\"firstnames\":[\"Joeran\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2021\",\"note\":\"Journal Abbreviation: CEUR-WS\",\"bibtex\":\"@inproceedings{scheidt_time-dependent_2021,\\n\\ttitle = {Time-dependent evaluation of recommender systems},\\n\\tvolume = {2955},\\n\\turl = {https://perspectives-ws.github.io/2021/proceedings/paper10.pdf},\\n\\turldate = {2021-09-27},\\n\\tbooktitle = {{CEUR}-{WS}},\\n\\tauthor = {Scheidt, Teresa and Beel, Joeran},\\n\\tmonth = sep,\\n\\tyear = {2021},\\n\\tnote = {Journal Abbreviation: CEUR-WS},\\n}\\n\\n\",\"author_short\":[\"Scheidt, T.\",\"Beel, J.\"],\"key\":\"scheidt_time-dependent_2021\",\"id\":\"scheidt_time-dependent_2021\",\"bibbaseid\":\"scheidt-beel-timedependentevaluationofrecommendersystems-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://perspectives-ws.github.io/2021/proceedings/paper10.pdf\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"title\":\"A Framework for Enhancing Deep Learning Based Recommender Systems with Knowledge Graphs\",\"url\":\"https://doi.org/10.1145/3472163.3472183\",\"doi\":\"10.1145/3472163.3472183\",\"abstract\":\"Recommendation methods fall into three major categories, content based filtering, collaborative filtering and deep learning based. Information about products and the preferences of earlier users are used in an unsupervised manner to create models which help make personalized recommendations to a specific new user. The more information we provide to these methods, the more likely it is that they yield better recommendations. Deep learning based methods are relatively recent, and are generally more robust to noise and missing information. This is because deep learning models can be trained even when some of the information records have partial information. Knowledge graphs represent the current trend in recording information in the form of relations between entities, and can provide any available information about products and users. This information is used to train the recommendation model. In this work, we present a new generic recommender systems framework, that integrates knowledge graphs into the recommendation pipeline. We describe its design and implementation, and then show through experiments, how such a framework can be specialized, taking the domain of movies as an example, and the resulting improvements in recommendations made possible by using all the information obtained using knowledge graphs. Our framework, to be made publicly available, supports different knowledge graph representation formats, and facilitates format conversion, merging and information extraction needed for training recommendation models.\",\"urldate\":\"2021-09-14\",\"booktitle\":\"IDEAS 2021\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Mudur\"],\"firstnames\":[\"Sudhir\",\"P\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mokhov\"],\"firstnames\":[\"Serguei\",\"A\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Mao\"],\"firstnames\":[\"Yuhao\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2021\",\"note\":\"Journal Abbreviation: IDEAS 2021\",\"keywords\":\"framework, knowledge graphs, recommendation model training, recommender system, deep learning based recommendations\",\"pages\":\"11–20\",\"bibtex\":\"@inproceedings{mudur_framework_2021,\\n\\taddress = {New York, NY, USA},\\n\\ttitle = {A {Framework} for {Enhancing} {Deep} {Learning} {Based} {Recommender} {Systems} with {Knowledge} {Graphs}},\\n\\turl = {https://doi.org/10.1145/3472163.3472183},\\n\\tdoi = {10.1145/3472163.3472183},\\n\\tabstract = {Recommendation methods fall into three major categories, content based\\nfiltering, collaborative filtering and deep learning based. Information\\nabout products and the preferences of earlier users are used in an\\nunsupervised manner to create models which help make personalized\\nrecommendations to a specific new user. The more information we provide to\\nthese methods, the more likely it is that they yield better\\nrecommendations. Deep learning based methods are relatively recent, and\\nare generally more robust to noise and missing information. This is\\nbecause deep learning models can be trained even when some of the\\ninformation records have partial information. Knowledge graphs represent\\nthe current trend in recording information in the form of relations\\nbetween entities, and can provide any available information about products\\nand users. This information is used to train the recommendation model. In\\nthis work, we present a new generic recommender systems framework, that\\nintegrates knowledge graphs into the recommendation pipeline. We describe\\nits design and implementation, and then show through experiments, how such\\na framework can be specialized, taking the domain of movies as an example,\\nand the resulting improvements in recommendations made possible by using\\nall the information obtained using knowledge graphs. Our framework, to be\\nmade publicly available, supports different knowledge graph representation\\nformats, and facilitates format conversion, merging and information\\nextraction needed for training recommendation models.},\\n\\turldate = {2021-09-14},\\n\\tbooktitle = {{IDEAS} 2021},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Mudur, Sudhir P and Mokhov, Serguei A and Mao, Yuhao},\\n\\tmonth = jul,\\n\\tyear = {2021},\\n\\tnote = {Journal Abbreviation: IDEAS 2021},\\n\\tkeywords = {framework, knowledge graphs, recommendation model training, recommender system, deep learning based recommendations},\\n\\tpages = {11--20},\\n}\\n\\n\",\"author_short\":[\"Mudur, S. P\",\"Mokhov, S. A\",\"Mao, Y.\"],\"key\":\"mudur_framework_2021\",\"id\":\"mudur_framework_2021\",\"bibbaseid\":\"mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3472163.3472183\"},\"keyword\":[\"framework\",\"knowledge graphs\",\"recommendation model training\",\"recommender system\",\"deep learning based recommendations\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"unpublished\",\"type\":\"unpublished\",\"title\":\"T-RECS: A simulation tool to study the societal impact of recommender systems\",\"url\":\"http://arxiv.org/abs/2107.08959\",\"abstract\":\"Simulation has emerged as a popular method to study the long-term societal consequences of recommender systems. This approach allows researchers to specify their theoretical model explicitly and observe the evolution of system-level outcomes over time. However, performing simulation-based studies often requires researchers to build their own simulation environments from the ground up, which creates a high barrier to entry, introduces room for implementation error, and makes it difficult to disentangle whether observed outcomes are due to the model or the implementation. We introduce T-RECS, an open-sourced Python package designed for researchers to simulate recommendation systems and other types of sociotechnical systems in which an algorithm mediates the interactions between multiple stakeholders, such as users and content creators. To demonstrate the flexibility of T-RECS, we perform a replication of two prior simulation-based research on sociotechnical systems. We additionally show how T-RECS can be used to generate novel insights with minimal overhead. Our tool promotes reproducibility in this area of research, provides a unified language for simulating sociotechnical systems, and removes the friction of implementing simulations from scratch.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Lucherini\"],\"firstnames\":[\"Eli\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Sun\"],\"firstnames\":[\"Matthew\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Winecoff\"],\"firstnames\":[\"Amy\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Narayanan\"],\"firstnames\":[\"Arvind\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2021\",\"note\":\"ISBN: 2107.08959 Publication Title: arXiv [cs.CY]\",\"bibtex\":\"@unpublished{lucherini_t-recs_2021,\\n\\ttitle = {T-{RECS}: {A} simulation tool to study the societal impact of recommender systems},\\n\\turl = {http://arxiv.org/abs/2107.08959},\\n\\tabstract = {Simulation has emerged as a popular method to study the long-term societal\\nconsequences of recommender systems. This approach allows researchers to\\nspecify their theoretical model explicitly and observe the evolution of\\nsystem-level outcomes over time. However, performing simulation-based\\nstudies often requires researchers to build their own simulation\\nenvironments from the ground up, which creates a high barrier to entry,\\nintroduces room for implementation error, and makes it difficult to\\ndisentangle whether observed outcomes are due to the model or the\\nimplementation. We introduce T-RECS, an open-sourced Python package\\ndesigned for researchers to simulate recommendation systems and other\\ntypes of sociotechnical systems in which an algorithm mediates the\\ninteractions between multiple stakeholders, such as users and content\\ncreators. To demonstrate the flexibility of T-RECS, we perform a\\nreplication of two prior simulation-based research on sociotechnical\\nsystems. We additionally show how T-RECS can be used to generate novel\\ninsights with minimal overhead. Our tool promotes reproducibility in this\\narea of research, provides a unified language for simulating\\nsociotechnical systems, and removes the friction of implementing\\nsimulations from scratch.},\\n\\tauthor = {Lucherini, Eli and Sun, Matthew and Winecoff, Amy and Narayanan, Arvind},\\n\\tmonth = jul,\\n\\tyear = {2021},\\n\\tnote = {ISBN: 2107.08959\\nPublication Title: arXiv [cs.CY]},\\n}\\n\\n\",\"author_short\":[\"Lucherini, E.\",\"Sun, M.\",\"Winecoff, A.\",\"Narayanan, A.\"],\"key\":\"lucherini_t-recs_2021\",\"id\":\"lucherini_t-recs_2021\",\"bibbaseid\":\"lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://arxiv.org/abs/2107.08959\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"address\":\"New York, NY, USA\",\"title\":\"New Insights into Metric Optimization for Ranking-based Recommendation\",\"url\":\"https://doi.org/10.1145/3404835.3462973\",\"doi\":\"10.1145/3404835.3462973\",\"abstract\":\"Direct optimization of IR metrics has often been adopted as an approach to devise and develop ranking-based recommender systems. Most methods following this approach (e.g. TFMAP, CLiMF, Top-N-Rank) aim at optimizing the same metric being used for evaluation, under the assumption that this will lead to the best performance. A number of studies of this practice bring this assumption, however, into question. In this paper, we dig deeper into this issue in order to learn more about the effects of the choice of the metric to optimize on the performance of a ranking-based recommender system. We present an extensive experimental study conducted on different datasets in both pairwise and listwise learning-to-rank (LTR) scenarios, to compare the relative merit of four popular IR metrics, namely RR, AP, nDCG and RBP, when used for optimization and assessment of recommender systems in various combinations. For the first three, we follow the practice of loss function formulation available in literature. For the fourth one, we propose novel loss functions inspired by RBP for both the pairwise and listwise scenario. Our results confirm that the best performance is indeed not necessarily achieved when optimizing the same metric being used for evaluation. In fact, we find that RBP-inspired losses perform at least as well as other metrics in a consistent way, and offer clear benefits in several cases. Interesting to see is that RBP-inspired losses, while improving the recommendation performance for all uses, may lead to an individual performance gain that is correlated with the activity level of a user in interacting with items. The more active the users, the more they benefit. Overall, our results challenge the assumption behind the current research practice of optimizing and evaluating the same metric, and point to RBP-based optimization instead as a promising alternative when learning to rank in the recommendation context.\",\"urldate\":\"2021-07-15\",\"booktitle\":\"SIGIR '21\",\"publisher\":\"Association for Computing Machinery\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Li\"],\"firstnames\":[\"Roger\",\"Zhe\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Urbano\"],\"firstnames\":[\"Julián\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Hanjalic\"],\"firstnames\":[\"Alan\"],\"suffixes\":[]}],\"month\":\"July\",\"year\":\"2021\",\"note\":\"Journal Abbreviation: SIGIR '21\",\"keywords\":\"evaluation metrics, learning to rank, recommender systems\",\"pages\":\"932–941\",\"bibtex\":\"@inproceedings{li_new_2021,\\n\\taddress = {New York, NY, USA},\\n\\ttitle = {New {Insights} into {Metric} {Optimization} for {Ranking}-based {Recommendation}},\\n\\turl = {https://doi.org/10.1145/3404835.3462973},\\n\\tdoi = {10.1145/3404835.3462973},\\n\\tabstract = {Direct optimization of IR metrics has often been adopted as an approach to\\ndevise and develop ranking-based recommender systems. Most methods\\nfollowing this approach (e.g. TFMAP, CLiMF, Top-N-Rank) aim at optimizing\\nthe same metric being used for evaluation, under the assumption that this\\nwill lead to the best performance. A number of studies of this practice\\nbring this assumption, however, into question. In this paper, we dig\\ndeeper into this issue in order to learn more about the effects of the\\nchoice of the metric to optimize on the performance of a ranking-based\\nrecommender system. We present an extensive experimental study conducted\\non different datasets in both pairwise and listwise learning-to-rank (LTR)\\nscenarios, to compare the relative merit of four popular IR metrics,\\nnamely RR, AP, nDCG and RBP, when used for optimization and assessment of\\nrecommender systems in various combinations. For the first three, we\\nfollow the practice of loss function formulation available in literature.\\nFor the fourth one, we propose novel loss functions inspired by RBP for\\nboth the pairwise and listwise scenario. Our results confirm that the best\\nperformance is indeed not necessarily achieved when optimizing the same\\nmetric being used for evaluation. In fact, we find that RBP-inspired\\nlosses perform at least as well as other metrics in a consistent way, and\\noffer clear benefits in several cases. Interesting to see is that\\nRBP-inspired losses, while improving the recommendation performance for\\nall uses, may lead to an individual performance gain that is correlated\\nwith the activity level of a user in interacting with items. The more\\nactive the users, the more they benefit. Overall, our results challenge\\nthe assumption behind the current research practice of optimizing and\\nevaluating the same metric, and point to RBP-based optimization instead as\\na promising alternative when learning to rank in the recommendation\\ncontext.},\\n\\turldate = {2021-07-15},\\n\\tbooktitle = {{SIGIR} '21},\\n\\tpublisher = {Association for Computing Machinery},\\n\\tauthor = {Li, Roger Zhe and Urbano, Julián and Hanjalic, Alan},\\n\\tmonth = jul,\\n\\tyear = {2021},\\n\\tnote = {Journal Abbreviation: SIGIR '21},\\n\\tkeywords = {evaluation metrics, learning to rank, recommender systems},\\n\\tpages = {932--941},\\n}\\n\\n\",\"author_short\":[\"Li, R. Z.\",\"Urbano, J.\",\"Hanjalic, A.\"],\"key\":\"li_new_2021\",\"id\":\"li_new_2021\",\"bibbaseid\":\"li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1145/3404835.3462973\"},\"keyword\":[\"evaluation metrics\",\"learning to rank\",\"recommender systems\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Fairness metrics and bias mitigation strategies for rating predictions\",\"volume\":\"58\",\"issn\":\"0306-4573\",\"url\":\"https://www.sciencedirect.com/science/article/pii/S0306457321001369\",\"doi\":\"10.1016/j.ipm.2021.102646\",\"abstract\":\"Algorithm fairness is an established line of research in the machine learning domain with substantial work while the equivalent in the recommender system domain is relatively new. In this article, we consider rating-based recommender systems which model the recommendation process as a prediction problem. We consider different types of biases that can occur in this setting, discuss various fairness definitions, and also propose a novel bias mitigation strategy to address potential unfairness in a rating-based recommender system. Based on an analysis of fairness metrics used in machine learning and a discussion of their applicability in the recommender system domain, we map the proposed metrics from the two domains and identify commonly used concepts and definitions of fairness. Finally, to address unfairness and potential bias against certain groups in a recommender system, we develop a bias mitigation algorithm and conduct case studies on one synthetic and one empirical dataset to show its effectiveness. Our results show that unfairness can be significantly lowered through our approach and that bias mitigation is a fruitful area of research for recommender systems.\",\"number\":\"5\",\"journal\":\"Inf. Process. Manag.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ashokan\"],\"firstnames\":[\"Ashwathy\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Haas\"],\"firstnames\":[\"Christian\"],\"suffixes\":[]}],\"month\":\"September\",\"year\":\"2021\",\"keywords\":\"Algorithmic fairness, Bias mitigation, Fairness metrics, Recommender systems\",\"pages\":\"102646\",\"bibtex\":\"@article{ashokan_fairness_2021,\\n\\ttitle = {Fairness metrics and bias mitigation strategies for rating predictions},\\n\\tvolume = {58},\\n\\tissn = {0306-4573},\\n\\turl = {https://www.sciencedirect.com/science/article/pii/S0306457321001369},\\n\\tdoi = {10.1016/j.ipm.2021.102646},\\n\\tabstract = {Algorithm fairness is an established line of research in the machine\\nlearning domain with substantial work while the equivalent in the\\nrecommender system domain is relatively new. In this article, we consider\\nrating-based recommender systems which model the recommendation process as\\na prediction problem. We consider different types of biases that can occur\\nin this setting, discuss various fairness definitions, and also propose a\\nnovel bias mitigation strategy to address potential unfairness in a\\nrating-based recommender system. Based on an analysis of fairness metrics\\nused in machine learning and a discussion of their applicability in the\\nrecommender system domain, we map the proposed metrics from the two\\ndomains and identify commonly used concepts and definitions of fairness.\\nFinally, to address unfairness and potential bias against certain groups\\nin a recommender system, we develop a bias mitigation algorithm and\\nconduct case studies on one synthetic and one empirical dataset to show\\nits effectiveness. Our results show that unfairness can be significantly\\nlowered through our approach and that bias mitigation is a fruitful area\\nof research for recommender systems.},\\n\\tnumber = {5},\\n\\tjournal = {Inf. Process. Manag.},\\n\\tauthor = {Ashokan, Ashwathy and Haas, Christian},\\n\\tmonth = sep,\\n\\tyear = {2021},\\n\\tkeywords = {Algorithmic fairness, Bias mitigation, Fairness metrics, Recommender systems},\\n\\tpages = {102646},\\n}\\n\\n\",\"author_short\":[\"Ashokan, A.\",\"Haas, C.\"],\"key\":\"ashokan_fairness_2021\",\"id\":\"ashokan_fairness_2021\",\"bibbaseid\":\"ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.sciencedirect.com/science/article/pii/S0306457321001369\"},\"keyword\":[\"Algorithmic fairness\",\"Bias mitigation\",\"Fairness metrics\",\"Recommender systems\"],\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Impact of End-User Privacy Enhancing Technologies (PETs) on Firms’ Analytics Performance\",\"url\":\"https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/\",\"abstract\":\"Big data analytics in digital commerce requires vast amounts of personal information from consumers, but this gives rise to major privacy concerns. To combat the threat of privacy invasion, individuals are proactively adopting privacy enhancing technologies (PETs) to protect their personal information. Consumers’ adoption of PETs may hamper firms’ big data analytics capabilities and performance but our knowledge of how PETs impact firms’ data analytics is limited. This study proposes an inductively derived framework which qualitatively shows that end-user PETs induce measurement error and/or missing values with regards to attributes, entities and relationships in firms’ customer databases, but the impacts of specific end-user PETs may vary by analytics use case. Our simulation experiments in the context of product recommendations quantitively find that consumers’ adoption characteristics (adoption rate and pattern) and PETs characteristics (protection mechanism and intensity) significantly affect the performance of recommender systems.\",\"urldate\":\"2021-01-28\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Chen\"],\"firstnames\":[\"Dawei\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Hahn\"],\"firstnames\":[\"Jungpil\"],\"suffixes\":[]}],\"year\":\"2020\",\"bibtex\":\"@inproceedings{chen_impact_2020,\\n\\ttitle = {Impact of {End}-{User} {Privacy} {Enhancing} {Technologies} ({PETs}) on {Firms}’ {Analytics} {Performance}},\\n\\turl = {https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/},\\n\\tabstract = {Big data analytics in digital commerce requires vast amounts of personal\\ninformation from consumers, but this gives rise to major privacy concerns.\\nTo combat the threat of privacy invasion, individuals are proactively\\nadopting privacy enhancing technologies (PETs) to protect their personal\\ninformation. Consumers’ adoption of PETs may hamper firms’ big data\\nanalytics capabilities and performance but our knowledge of how PETs\\nimpact firms’ data analytics is limited. This study proposes an\\ninductively derived framework which qualitatively shows that end-user PETs\\ninduce measurement error and/or missing values with regards to attributes,\\nentities and relationships in firms’ customer databases, but the impacts\\nof specific end-user PETs may vary by analytics use case. Our simulation\\nexperiments in the context of product recommendations quantitively find\\nthat consumers’ adoption characteristics (adoption rate and pattern) and\\nPETs characteristics (protection mechanism and intensity) significantly\\naffect the performance of recommender systems.},\\n\\turldate = {2021-01-28},\\n\\tauthor = {Chen, Dawei and Hahn, Jungpil},\\n\\tyear = {2020},\\n}\\n\\n\",\"author_short\":[\"Chen, D.\",\"Hahn, J.\"],\"key\":\"chen_impact_2020\",\"id\":\"chen_impact_2020\",\"bibbaseid\":\"chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":0,\"html\":\"\"},{\"bibtype\":\"mastersthesis\",\"type\":\"mastersthesis\",\"title\":\"What is the Value of Rating Obscure Items? An Analysis of the Effect of Less-Popular Items on Recommendation Quality\",\"url\":\"https://conservancy.umn.edu/handle/11299/206148\",\"abstract\":\"Recommender systems designers believe that the system stands to benefit from the users rating items that do not have many ratings. However, the effect of this act of rating lesser known items on the user's recommendations is unknown. This leads to asking the question of whether these low popularity items affect the recommendations received by users. This work looks at the effect less popular items have on a user's recommendations and the prediction and recommendations metrics that quantify the quality of recommendations …\",\"school\":\"University of Minnesota\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Narayan\"],\"firstnames\":[\"A\"],\"suffixes\":[]}],\"year\":\"2019\",\"note\":\"Publication Title: Department of Computer Science Volume: M.S. in Computer Science\",\"bibtex\":\"@mastersthesis{narayan_what_2019,\\n\\ttitle = {What is the {Value} of {Rating} {Obscure} {Items}? {An} {Analysis} of the {Effect} of {Less}-{Popular} {Items} on {Recommendation} {Quality}},\\n\\turl = {https://conservancy.umn.edu/handle/11299/206148},\\n\\tabstract = {Recommender systems designers believe that the system stands to benefit\\nfrom the users rating items that do not have many ratings. However, the\\neffect of this act of rating lesser known items on the user's\\nrecommendations is unknown. This leads to asking the question of whether\\nthese low popularity items affect the recommendations received by users.\\nThis work looks at the effect less popular items have on a user's\\nrecommendations and the prediction and recommendations metrics that\\nquantify the quality of recommendations …},\\n\\tschool = {University of Minnesota},\\n\\tauthor = {Narayan, A},\\n\\tyear = {2019},\\n\\tnote = {Publication Title: Department of Computer Science\\nVolume: M.S. in Computer Science},\\n}\\n\\n\",\"author_short\":[\"Narayan, A\"],\"key\":\"narayan_what_2019\",\"id\":\"narayan_what_2019\",\"bibbaseid\":\"narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://conservancy.umn.edu/handle/11299/206148\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":2,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Using facial recognition services as implicit feedback for recommenders\",\"volume\":\"2450\",\"url\":\"http://ceur-ws.org/Vol-2450/paper4.pdf\",\"abstract\":\"User authentication and feedback gathering are crucial aspects for recommender systems. The most common implementations, a username / password login and star rating systems, require user interaction and a cognitive effort from the user. As a result, users opt to save their password in the interface and optional feedback with a star rating system is often skipped, especially for applications such as video watching in a home environment. In this article, we propose an alternative method for user authentication based on facial recognition and an automatic feedback gathering method by detecting various face characteristics. Using facial recognition with a camera in a tablet, smartphone, or smart TV, the persons in front of the screen can be identified in order to link video watching sessions to their user profile. During video watching, implicit feedback is automatically gathered through emotion recognition, attention measurements, and behavior analysis. An emotion fingerprint, which is defined as a unique spectrum of expected emotions for a video scene, is compared to the recognized emotions in order to estimate the experience of a user while watching. An evaluation with a test panel showed that happiness can be most accurately detected and the recognized emotions are correlated with the user’s star rating.\",\"publisher\":\"CEUR-WS\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Pessemier\"],\"firstnames\":[\"Toon\",\"De\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Coppens\"],\"firstnames\":[\"Ine\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Martens\"],\"firstnames\":[\"Luc\"],\"suffixes\":[]}],\"year\":\"2019\",\"pages\":\"8\",\"bibtex\":\"@inproceedings{pessemier_using_2019,\\n\\ttitle = {Using facial recognition services as implicit feedback for recommenders},\\n\\tvolume = {2450},\\n\\turl = {http://ceur-ws.org/Vol-2450/paper4.pdf},\\n\\tabstract = {User authentication and feedback gathering are crucial aspects for\\nrecommender systems. The most common implementations, a username /\\npassword login and star rating systems, require user interaction and a\\ncognitive effort from the user. As a result, users opt to save their\\npassword in the interface and optional feedback with a star rating system\\nis often skipped, especially for applications such as video watching in a\\nhome environment. In this article, we propose an alternative method for\\nuser authentication based on facial recognition and an automatic feedback\\ngathering method by detecting various face characteristics. Using facial\\nrecognition with a camera in a tablet, smartphone, or smart TV, the\\npersons in front of the screen can be identified in order to link video\\nwatching sessions to their user profile. During video watching, implicit\\nfeedback is automatically gathered through emotion recognition, attention\\nmeasurements, and behavior analysis. An emotion fingerprint, which is\\ndefined as a unique spectrum of expected emotions for a video scene, is\\ncompared to the recognized emotions in order to estimate the experience of\\na user while watching. An evaluation with a test panel showed that\\nhappiness can be most accurately detected and the recognized emotions are\\ncorrelated with the user’s star rating.},\\n\\tpublisher = {CEUR-WS},\\n\\tauthor = {Pessemier, Toon De and Coppens, Ine and Martens, Luc},\\n\\tyear = {2019},\\n\\tpages = {8},\\n}\\n\\n\",\"author_short\":[\"Pessemier, T. D.\",\"Coppens, I.\",\"Martens, L.\"],\"key\":\"pessemier_using_2019\",\"id\":\"pessemier_using_2019\",\"bibbaseid\":\"pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://ceur-ws.org/Vol-2450/paper4.pdf\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":3,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Standing in Your Shoes: External Assessments for Personalized Recommender Systems\",\"url\":\"http://dx.doi.org/10.1145/3404835.3462916\",\"doi\":\"10.1145/3404835.3462916\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Lu\"],\"firstnames\":[\"Hongyu\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ma\"],\"firstnames\":[\"Weizhi\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Zhang\"],\"firstnames\":[\"Min\"],\"suffixes\":[]},{\"propositions\":[\"de\"],\"lastnames\":[\"Rijke\"],\"firstnames\":[\"Maarten\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Liu\"],\"firstnames\":[\"Yiqun\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Ma\"],\"firstnames\":[\"Shaoping\"],\"suffixes\":[]}],\"year\":\"2021\",\"bibtex\":\"@inproceedings{lu_standing_2021,\\n\\ttitle = {Standing in {Your} {Shoes}: {External} {Assessments} for {Personalized} {Recommender} {Systems}},\\n\\turl = {http://dx.doi.org/10.1145/3404835.3462916},\\n\\tdoi = {10.1145/3404835.3462916},\\n\\tauthor = {Lu, Hongyu and Ma, Weizhi and Zhang, Min and de Rijke, Maarten and Liu, Yiqun and Ma, Shaoping},\\n\\tyear = {2021},\\n}\\n\\n\",\"author_short\":[\"Lu, H.\",\"Ma, W.\",\"Zhang, M.\",\"de Rijke, M.\",\"Liu, Y.\",\"Ma, S.\"],\"key\":\"lu_standing_2021\",\"id\":\"lu_standing_2021\",\"bibbaseid\":\"lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://dx.doi.org/10.1145/3404835.3462916\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":21,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"A Recommender Systems’ algorithm evaluation using the Lenskit library and MovieLens databases\",\"url\":\"http://dx.doi.org/10.1109/BMSB49480.2020.9379914\",\"doi\":\"10.1109/BMSB49480.2020.9379914\",\"abstract\":\"A wide variety of algorithms can be found in the recommender system's literature for computing predictions and obtaining recommendation lists. These algorithms' performance can be evaluated by numerous metrics depending on which business goal is intended to be optimized. However, performance can vary drastically depending on the selected algorithm as well as the selected metric. In addition, algorithmic performance can be heavily influenced by the experimental setup such as: dataset, cross-validation strategy, recommender framework and the model hyper-parameters chosen. In this work we will compare nine different algorithms used for generating recommendations across seven different metrics. One vital aspect of our analysis is to provide a detailed and clear explanation of the methodology carried out as well as the experimental setup in order to ensure future reproducibility. Moreover, performance and computation times will be introduced for further comparison. Finally, optimal model hyper-parameter combinations will be presented for benchmarking purposes.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Paullier\"],\"firstnames\":[\"Alejo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Sotelo\"],\"firstnames\":[\"Rafael\"],\"suffixes\":[]}],\"month\":\"October\",\"year\":\"2020\",\"keywords\":\"Algorithms, Business, Computational modeling, Databases, Evaluation, LensKit, Libraries, MovieLens, Multimedia communication, Prediction algorithms, Recommender systems, Reproducibility of results\",\"pages\":\"1–7\",\"bibtex\":\"@inproceedings{paullier_recommender_2020,\\n\\ttitle = {A {Recommender} {Systems}’ algorithm evaluation using the {Lenskit} library and {MovieLens} databases},\\n\\turl = {http://dx.doi.org/10.1109/BMSB49480.2020.9379914},\\n\\tdoi = {10.1109/BMSB49480.2020.9379914},\\n\\tabstract = {A wide variety of algorithms can be found in the recommender system's\\nliterature for computing predictions and obtaining recommendation lists.\\nThese algorithms' performance can be evaluated by numerous metrics\\ndepending on which business goal is intended to be optimized. However,\\nperformance can vary drastically depending on the selected algorithm as\\nwell as the selected metric. In addition, algorithmic performance can be\\nheavily influenced by the experimental setup such as: dataset,\\ncross-validation strategy, recommender framework and the model\\nhyper-parameters chosen. In this work we will compare nine different\\nalgorithms used for generating recommendations across seven different\\nmetrics. One vital aspect of our analysis is to provide a detailed and\\nclear explanation of the methodology carried out as well as the\\nexperimental setup in order to ensure future reproducibility. Moreover,\\nperformance and computation times will be introduced for further\\ncomparison. Finally, optimal model hyper-parameter combinations will be\\npresented for benchmarking purposes.},\\n\\tauthor = {Paullier, Alejo and Sotelo, Rafael},\\n\\tmonth = oct,\\n\\tyear = {2020},\\n\\tkeywords = {Algorithms, Business, Computational modeling, Databases, Evaluation, LensKit, Libraries, MovieLens, Multimedia communication, Prediction algorithms, Recommender systems, Reproducibility of results},\\n\\tpages = {1--7},\\n}\\n\\n\",\"author_short\":[\"Paullier, A.\",\"Sotelo, R.\"],\"key\":\"paullier_recommender_2020\",\"id\":\"paullier_recommender_2020\",\"bibbaseid\":\"paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://dx.doi.org/10.1109/BMSB49480.2020.9379914\"},\"keyword\":[\"Algorithms\",\"Business\",\"Computational modeling\",\"Databases\",\"Evaluation\",\"LensKit\",\"Libraries\",\"MovieLens\",\"Multimedia communication\",\"Prediction algorithms\",\"Recommender systems\",\"Reproducibility of results\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"mastersthesis\",\"type\":\"mastersthesis\",\"title\":\"Sistema de recomendación de matrícula de cursos electivos para estudiantes de Ingeniería Electrónica e Ingeniería de Telecomunicaciones de la UNAD\",\"url\":\"https://repository.unad.edu.co/handle/10596/40465\",\"urldate\":\"2021-05-12\",\"school\":\"Universidad Nacional Abierta y a Distancia\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Noguera\",\"Torres\"],\"firstnames\":[\"Adriana\",\"del\",\"Pilar\"],\"suffixes\":[]}],\"collaborator\":\"Rúa Pérez, Santiago\",\"month\":\"April\",\"year\":\"2021\",\"note\":\"Volume: MS in Information Technology\",\"bibtex\":\"@mastersthesis{noguera_torres_sistema_2021,\\n\\ttitle = {Sistema de recomendación de matrícula de cursos electivos para estudiantes de {Ingeniería} {Electrónica} e {Ingeniería} de {Telecomunicaciones} de la {UNAD}},\\n\\turl = {https://repository.unad.edu.co/handle/10596/40465},\\n\\turldate = {2021-05-12},\\n\\tschool = {Universidad Nacional Abierta y a Distancia},\\n\\tauthor = {Noguera Torres, Adriana del Pilar},\\n\\tcollaborator = {Rúa Pérez, Santiago},\\n\\tmonth = apr,\\n\\tyear = {2021},\\n\\tnote = {Volume: MS in Information Technology},\\n}\\n\\n\",\"author_short\":[\"Noguera Torres, A. d. P.\"],\"key\":\"noguera_torres_sistema_2021\",\"id\":\"noguera_torres_sistema_2021\",\"bibbaseid\":\"nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://repository.unad.edu.co/handle/10596/40465\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"incollection\",\"type\":\"incollection\",\"address\":\"Wiesbaden\",\"title\":\"Empfehlungssysteme\",\"isbn\":\"978-3-658-17291-6\",\"url\":\"https://doi.org/10.1007/978-3-658-17291-6_52\",\"abstract\":\"Empfehlungssysteme stellen heute eine zentrale Komponente vieler Online-Plattformen dar, die bei Online-Shops und vielen anderen Anwendungen häufig zum Einsatz kommt. Ziel ist es, dem Kunden entsprechend seinen persönlichen Präferenzen Produkte oder andere Artikel vorzuschlagen, die für ihn von Interesse sind und potenziell zu einem Kauf oder generell zur Nutzung führen. Empfehlungssysteme haben eine erhebliche wirtschaftliche Bedeutung, da sie in vielen Fällen zu einem signifikanten Anteil zu Erfolgsfaktoren wie Click-through-Raten oder Käufen beitragen. Wir stellen in diesem Kapitel die unterschiedlichen Ansätze zur automatisierten Empfehlungsgebung vor und beschreiben konkrete Techniken zu deren Umsetzung. Weiterhin gehen wir auf wesentliche Aspekte der Gestaltung und Bewertung von Empfehlungssystemen ein und diskutieren anwenderrelevante Themen wie Usability und Vertrauen in systemgenerierte Empfehlungen.\",\"booktitle\":\"Handbuch Digitale Wirtschaft\",\"publisher\":\"Springer Fachmedien Wiesbaden\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ziegler\"],\"firstnames\":[\"Jürgen\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Loepp\"],\"firstnames\":[\"Benedikt\"],\"suffixes\":[]}],\"editor\":[{\"propositions\":[],\"lastnames\":[\"Kollmann\"],\"firstnames\":[\"Tobias\"],\"suffixes\":[]}],\"year\":\"2020\",\"doi\":\"10.1007/978-3-658-17291-6_52\",\"pages\":\"717–741\",\"bibtex\":\"@incollection{ziegler_empfehlungssysteme_2020,\\n\\taddress = {Wiesbaden},\\n\\ttitle = {Empfehlungssysteme},\\n\\tisbn = {978-3-658-17291-6},\\n\\turl = {https://doi.org/10.1007/978-3-658-17291-6_52},\\n\\tabstract = {Empfehlungssysteme stellen heute eine zentrale Komponente vieler\\nOnline-Plattformen dar, die bei Online-Shops und vielen anderen\\nAnwendungen häufig zum Einsatz kommt. Ziel ist es, dem Kunden entsprechend\\nseinen persönlichen Präferenzen Produkte oder andere Artikel\\nvorzuschlagen, die für ihn von Interesse sind und potenziell zu einem Kauf\\noder generell zur Nutzung führen. Empfehlungssysteme haben eine erhebliche\\nwirtschaftliche Bedeutung, da sie in vielen Fällen zu einem signifikanten\\nAnteil zu Erfolgsfaktoren wie Click-through-Raten oder Käufen beitragen.\\nWir stellen in diesem Kapitel die unterschiedlichen Ansätze zur\\nautomatisierten Empfehlungsgebung vor und beschreiben konkrete Techniken\\nzu deren Umsetzung. Weiterhin gehen wir auf wesentliche Aspekte der\\nGestaltung und Bewertung von Empfehlungssystemen ein und diskutieren\\nanwenderrelevante Themen wie Usability und Vertrauen in systemgenerierte\\nEmpfehlungen.},\\n\\tbooktitle = {Handbuch {Digitale} {Wirtschaft}},\\n\\tpublisher = {Springer Fachmedien Wiesbaden},\\n\\tauthor = {Ziegler, Jürgen and Loepp, Benedikt},\\n\\teditor = {Kollmann, Tobias},\\n\\tyear = {2020},\\n\\tdoi = {10.1007/978-3-658-17291-6_52},\\n\\tpages = {717--741},\\n}\\n\\n\",\"author_short\":[\"Ziegler, J.\",\"Loepp, B.\"],\"editor_short\":[\"Kollmann, T.\"],\"key\":\"ziegler_empfehlungssysteme_2020\",\"id\":\"ziegler_empfehlungssysteme_2020\",\"bibbaseid\":\"ziegler-loepp-empfehlungssysteme-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1007/978-3-658-17291-6_52\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":1,\"html\":\"\"},{\"bibtype\":\"inproceedings\",\"type\":\"inproceedings\",\"title\":\"Bayesian Deep Learning Based Exploration-Exploitation for Personalized Recommendations\",\"url\":\"http://dx.doi.org/10.1109/ICTAI.2019.00253\",\"doi\":\"10.1109/ICTAI.2019.00253\",\"abstract\":\"Personalized Recommendation Systems require an effective method to balance exploration and exploitation. To learn effective strategies, user and item attributes are critical data sources to capture contextual information. In this paper, we first present an approach based on Bayesian Deep Learning to learn a compact representation of user and item attributes to guide exploitation. A key novelty of the approach lies in its ability to also capture the uncertainty associated with the model output to guide exploration. We then show how to further boost exploration by incorporating model uncertainty with that of data uncertainty. Experimental results demonstrate the benefits of our approach in terms of accuracy in recommendations as well as its performance in an online setting.\",\"publisher\":\"ieeexplore.ieee.org\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Wang\"],\"firstnames\":[\"X\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Kadioglu\"],\"firstnames\":[\"S\"],\"suffixes\":[]}],\"month\":\"November\",\"year\":\"2019\",\"keywords\":\"Bayesian Deep Learning, ExplorationExploitation, Personalized Recommendation, Bayesian deep learning, boost exploration, compact representation, contextual information, critical data sources, data uncertainty, exploration-exploitation, information filtering, item attributes, learning (artificial intelligence), model output, model uncertainty, personalized recommendation systems, recommender systems\",\"pages\":\"1715–1719\",\"bibtex\":\"@inproceedings{wang_bayesian_2019,\\n\\ttitle = {Bayesian {Deep} {Learning} {Based} {Exploration}-{Exploitation} for {Personalized} {Recommendations}},\\n\\turl = {http://dx.doi.org/10.1109/ICTAI.2019.00253},\\n\\tdoi = {10.1109/ICTAI.2019.00253},\\n\\tabstract = {Personalized Recommendation Systems require an effective method to balance\\nexploration and exploitation. To learn effective strategies, user and item\\nattributes are critical data sources to capture contextual information. In\\nthis paper, we first present an approach based on Bayesian Deep Learning\\nto learn a compact representation of user and item attributes to guide\\nexploitation. A key novelty of the approach lies in its ability to also\\ncapture the uncertainty associated with the model output to guide\\nexploration. We then show how to further boost exploration by\\nincorporating model uncertainty with that of data uncertainty.\\nExperimental results demonstrate the benefits of our approach in terms of\\naccuracy in recommendations as well as its performance in an online\\nsetting.},\\n\\tpublisher = {ieeexplore.ieee.org},\\n\\tauthor = {Wang, X and Kadioglu, S},\\n\\tmonth = nov,\\n\\tyear = {2019},\\n\\tkeywords = {Bayesian Deep Learning, ExplorationExploitation, Personalized Recommendation, Bayesian deep learning, boost exploration, compact representation, contextual information, critical data sources, data uncertainty, exploration-exploitation, information filtering, item attributes, learning (artificial intelligence), model output, model uncertainty, personalized recommendation systems, recommender systems},\\n\\tpages = {1715--1719},\\n}\\n\\n\",\"author_short\":[\"Wang, X\",\"Kadioglu, S\"],\"key\":\"wang_bayesian_2019\",\"id\":\"wang_bayesian_2019\",\"bibbaseid\":\"wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://dx.doi.org/10.1109/ICTAI.2019.00253\"},\"keyword\":[\"Bayesian Deep Learning\",\"ExplorationExploitation\",\"Personalized Recommendation\",\"Bayesian deep learning\",\"boost exploration\",\"compact representation\",\"contextual information\",\"critical data sources\",\"data uncertainty\",\"exploration-exploitation\",\"information filtering\",\"item attributes\",\"learning (artificial intelligence)\",\"model output\",\"model uncertainty\",\"personalized recommendation systems\",\"recommender systems\"],\"metadata\":{\"authorlinks\":{}},\"downloads\":4,\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"Evaluating facial recognition services as interaction technique for recommender systems\",\"issn\":\"1380-7501\",\"url\":\"https://doi.org/10.1007/s11042-020-09061-8\",\"doi\":\"10.1007/s11042-020-09061-8\",\"abstract\":\"Recommender systems are tools and techniques to assist users in the content selection process thereby coping with the problem of information overload. For recommender systems, user authentication and feedback gathering are of crucial importance. However, the typical user authentication with username / password and feedback method with a star rating system are not user friendly and often bypassed. This article proposes an alternative method for user authentication based on facial recognition and an automatic feedback gathering method by detecting various face characteristics such as emotions. We studied the use case of video watching. Photos made with the front-facing camera of a tablet, smartphone, or smart TV are used as input of a facial recognition service. The persons in front of the screen can be identified. During video watching, implicit feedback for the video content is automatically gathered through emotion recognition, attention measurements, and behavior analysis. An evaluation with a test panel showed that the recognized emotions are correlated with the user’s star ratings and that happiness can be most accurately detected. So as the main contribution, this article indicates that emotion recognition might be used as an alternative feedback mechanism for recommender systems.\",\"journal\":\"Multimed. Tools Appl.\",\"author\":[{\"propositions\":[],\"lastnames\":[\"De\",\"Pessemier\"],\"firstnames\":[\"Toon\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Coppens\"],\"firstnames\":[\"Ine\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Martens\"],\"firstnames\":[\"Luc\"],\"suffixes\":[]}],\"month\":\"June\",\"year\":\"2020\",\"bibtex\":\"@article{de_pessemier_evaluating_2020,\\n\\ttitle = {Evaluating facial recognition services as interaction technique for recommender systems},\\n\\tissn = {1380-7501},\\n\\turl = {https://doi.org/10.1007/s11042-020-09061-8},\\n\\tdoi = {10.1007/s11042-020-09061-8},\\n\\tabstract = {Recommender systems are tools and techniques to assist users in the\\ncontent selection process thereby coping with the problem of information\\noverload. For recommender systems, user authentication and feedback\\ngathering are of crucial importance. However, the typical user\\nauthentication with username / password and feedback method with a star\\nrating system are not user friendly and often bypassed. This article\\nproposes an alternative method for user authentication based on facial\\nrecognition and an automatic feedback gathering method by detecting\\nvarious face characteristics such as emotions. We studied the use case of\\nvideo watching. Photos made with the front-facing camera of a tablet,\\nsmartphone, or smart TV are used as input of a facial recognition service.\\nThe persons in front of the screen can be identified. During video\\nwatching, implicit feedback for the video content is automatically\\ngathered through emotion recognition, attention measurements, and behavior\\nanalysis. An evaluation with a test panel showed that the recognized\\nemotions are correlated with the user’s star ratings and that happiness\\ncan be most accurately detected. So as the main contribution, this article\\nindicates that emotion recognition might be used as an alternative\\nfeedback mechanism for recommender systems.},\\n\\tjournal = {Multimed. Tools Appl.},\\n\\tauthor = {De Pessemier, Toon and Coppens, Ine and Martens, Luc},\\n\\tmonth = jun,\\n\\tyear = {2020},\\n}\\n\\n\",\"author_short\":[\"De Pessemier, T.\",\"Coppens, I.\",\"Martens, L.\"],\"key\":\"de_pessemier_evaluating_2020\",\"id\":\"de_pessemier_evaluating_2020\",\"bibbaseid\":\"depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://doi.org/10.1007/s11042-020-09061-8\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":5,\"html\":\"\"},{\"bibtype\":\"incollection\",\"type\":\"incollection\",\"title\":\"Recommender Systems\",\"isbn\":\"978-1-4842-4858-4\",\"url\":\"http://dx.doi.org/10.1007/978-1-4842-4859-1\",\"booktitle\":\"Practical Data Science with Python 3\",\"publisher\":\"Apress\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Varga\"],\"firstnames\":[\"Ervin\"],\"suffixes\":[]}],\"year\":\"2019\",\"doi\":\"10.1007/978-1-4842-4859-1\",\"pages\":\"317–339\",\"bibtex\":\"@incollection{varga_recommender_2019,\\n\\ttitle = {Recommender {Systems}},\\n\\tisbn = {978-1-4842-4858-4},\\n\\turl = {http://dx.doi.org/10.1007/978-1-4842-4859-1},\\n\\tbooktitle = {Practical {Data} {Science} with {Python} 3},\\n\\tpublisher = {Apress},\\n\\tauthor = {Varga, Ervin},\\n\\tyear = {2019},\\n\\tdoi = {10.1007/978-1-4842-4859-1},\\n\\tpages = {317--339},\\n}\\n\\n\",\"author_short\":[\"Varga, E.\"],\"key\":\"varga_recommender_2019\",\"id\":\"varga_recommender_2019\",\"bibbaseid\":\"varga-recommendersystems-2019\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://dx.doi.org/10.1007/978-1-4842-4859-1\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":5,\"html\":\"\"},{\"bibtype\":\"mastersthesis\",\"type\":\"mastersthesis\",\"title\":\"Estudio del estado del arte de la ciencia de datos aplicada a la neuroeconomía\",\"url\":\"http://dspace.ups.edu.ec/handle/123456789/17605\",\"school\":\"Universidad Politécnica Salesiana\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Balseca\",\"Níñez\"],\"firstnames\":[\"Christian\",\"Ricardo\"],\"suffixes\":[]},{\"propositions\":[],\"lastnames\":[\"Fernández\",\"Peñafiel\"],\"firstnames\":[\"Daniel\",\"Andrés\"],\"suffixes\":[]}],\"year\":\"2019\",\"bibtex\":\"@mastersthesis{balseca_ninez_estudio_2019,\\n\\ttitle = {Estudio del estado del arte de la ciencia de datos aplicada a la neuroeconomía},\\n\\turl = {http://dspace.ups.edu.ec/handle/123456789/17605},\\n\\tschool = {Universidad Politécnica Salesiana},\\n\\tauthor = {Balseca Níñez, Christian Ricardo and Fernández Peñafiel, Daniel Andrés},\\n\\tyear = {2019},\\n}\\n\\n\",\"author_short\":[\"Balseca Níñez, C. R.\",\"Fernández Peñafiel, D. A.\"],\"key\":\"balseca_ninez_estudio_2019\",\"id\":\"balseca_ninez_estudio_2019\",\"bibbaseid\":\"balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019\",\"role\":\"author\",\"urls\":{\"Paper\":\"http://dspace.ups.edu.ec/handle/123456789/17605\"},\"metadata\":{\"authorlinks\":{}},\"html\":\"\"},{\"bibtype\":\"article\",\"type\":\"article\",\"title\":\"CBRec: a book recommendation system for children using the matrix factorisation and content-based filtering approaches\",\"volume\":\"16\",\"issn\":\"1743-8187\",\"url\":\"https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738\",\"doi\":\"10.1504/IJBIDM.2020.104738\",\"abstract\":\"Promoting good reading habits among children is essential, given the enormous influence of reading on students' development as learners and members of the society. Unfortunately, very few (children) websites or online applications recommend books to children, even though they can play a significant role in encouraging children to read. Given that a few popular book websites suggest books to children based on the popularity of books or rankings on books, they are not customised/personalised for each individual user and likely recommend books that users do not want or like. We have integrated the matrix factorisation approach and the content-based approach, in addition to predicting the grade levels of books, to recommend books for children. Recent research works have demonstrated that a hybrid approach, which combines different filtering approaches, is more effective in making recommendations. Conducted empirical study has verified the effectiveness of our proposed children book recommendation system.\",\"number\":\"2\",\"journal\":\"International Journal of Business Intelligence and Data Mining\",\"author\":[{\"propositions\":[],\"lastnames\":[\"Ng\"],\"firstnames\":[\"Yiu-Kai\"],\"suffixes\":[]}],\"month\":\"January\",\"year\":\"2020\",\"note\":\"Publisher: Inderscience Publishers\",\"pages\":\"129–149\",\"bibtex\":\"@article{ng_cbrec_2020,\\n\\ttitle = {{CBRec}: a book recommendation system for children using the matrix factorisation and content-based filtering approaches},\\n\\tvolume = {16},\\n\\tissn = {1743-8187},\\n\\turl = {https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738},\\n\\tdoi = {10.1504/IJBIDM.2020.104738},\\n\\tabstract = {Promoting good reading habits among children is essential, given the\\nenormous influence of reading on students' development as learners and\\nmembers of the society. Unfortunately, very few (children) websites or\\nonline applications recommend books to children, even though they can play\\na significant role in encouraging children to read. Given that a few\\npopular book websites suggest books to children based on the popularity of\\nbooks or rankings on books, they are not customised/personalised for each\\nindividual user and likely recommend books that users do not want or like.\\nWe have integrated the matrix factorisation approach and the content-based\\napproach, in addition to predicting the grade levels of books, to\\nrecommend books for children. Recent research works have demonstrated that\\na hybrid approach, which combines different filtering approaches, is more\\neffective in making recommendations. Conducted empirical study has\\nverified the effectiveness of our proposed children book recommendation\\nsystem.},\\n\\tnumber = {2},\\n\\tjournal = {International Journal of Business Intelligence and Data Mining},\\n\\tauthor = {Ng, Yiu-Kai},\\n\\tmonth = jan,\\n\\tyear = {2020},\\n\\tnote = {Publisher: Inderscience Publishers},\\n\\tpages = {129--149},\\n}\\n\",\"author_short\":[\"Ng, Y.\"],\"key\":\"ng_cbrec_2020\",\"id\":\"ng_cbrec_2020\",\"bibbaseid\":\"ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020\",\"role\":\"author\",\"urls\":{\"Paper\":\"https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738\"},\"metadata\":{\"authorlinks\":{}},\"downloads\":9,\"html\":\"\"}],\n      metadata: {\"authorlinks\":{}},\n      ownerID: undefined\n    };\n  </script>\n\n  <script type=\"text/javascript\">\n  var site$;\n  if (typeof $ != 'undefined') {\n    console.log('existing jquery: storing and then restoring');\n    site$ = $;\n    $ = null;\n  }\n  </script>\n\n  <script src=\"https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js\">\n  </script>\n  <script type=\"text/javascript\">\n  if (site$) {\n    console.log('existing jquery: avoiding conflict and restoring');\n    bb$ = $.noConflict(true);\n    $ = site$;\n  } else {\n    bb$ = $;\n  }\n  </script>\n\n  <script src=\"//bibbase.org/js/bibbase.min.js\"\n          type=\"text/javascript\">\n  </script>\n\n  <script type=\"text/javascript\"\n          src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML\">\n  </script>\n  <script type=\"text/x-mathjax-config\">\n    MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\\\(','\\\\)']]},\n                        skipTags: [\"body\"],\n                        processClass: \"bibbase_paper\"});\n  </script>\n\n  <style>\n  @media print {\n    .dontprint {\n        display: none !important;\n    }\n\n  .bibbase_group {\n    background-color: #E0E0E0;\n    font-style: italic;\n    font-size: larger;\n    text-shadow: 0px 0px 3px #FFFFFF;\n    border-radius: 4px 4px 4px 4px;\n    -moz-border-radius: 4px 4px 4px 4px;\n    -webkit-border-radius: 4px;\n    padding: 5px 40px 5px;\n    margin-top: 10px;\n    margin-bottom: 5px;\n    cursor: pointer;\n  }\n\n  .bibbase_paper {\n    margin-bottom: 5px;\n  }\n\n  }\n  </style>\n\n  \n  <div style=\"float: right; margin-right: 10px; margin-top: 10px; text-align: right; font-size: 12px\">\n    generated by\n    <a href=\"//bibbase.org\"\n       onclick=\"trackOutboundLink('bibbase', 'logo clicked', window.location.href, '//bibbase.org'); return false;\">\n      <img src=\"//bibbase.org/img/logo.svg\"\n           style=\"vertical-align: middle; margin-left: 3px; height: 16px;\"/\n           alt=\"bibbase.org\"\n           title=\"BibBase – The easiest way to maintain your publications page.\"\n        ></a>\n    \n  </div>\n  \n\n  <div id=\"bibbase_header\" class=\"dontprint\" style=\"\">\n\n    <ul class=\"nav nav-pills\" style=\"margin: 0px;\">\n\n      <li class=\"dropdown\" id=\"groupby_dropdown\">\n      <a class=\"dropdown-toggle\"\n         data-toggle=\"dropdown\"\n         href=\"#\">\n          Group by\n          <b class=\"caret\"></b>\n      </a>\n      <ul class=\"dropdown-menu\">\n        <li class=\"groupby year\"><a onclick=\"groupby('year')\">\n            Year</a>\n        </li>\n        <li class=\"groupby author\"><a onclick=\"groupby('author_short')\">\n            Author</a>\n        </li>\n        <li class=\"groupby type\"><a onclick=\"groupby('type')\">\n            Type</a>\n        </li>\n        <li class=\"groupby keyword\"><a onclick=\"groupby('keyword')\">\n            Keyword</a>\n        </li>\n        <li class=\"groupby downloads\"><a onclick=\"groupby('downloads')\">\n            Downloads</a>\n        </li>\n      </ul>\n      </li>\n\n\n      <li class=\"dropdown\" id=\"menu_dropdown\">\n      <a class=\"dropdown-toggle\"\n         data-toggle=\"dropdown\"\n         href=\"#\" alt=\"controls\">\n          <i class=\"fa fa-reorder\" alt=\"controls\"></i>\n          <b class=\"caret\"></b>\n      </a>\n      <ul class=\"dropdown-menu\">\n        <li>\n          <a onclick=\"toggleAll()\">\n            <i class=\"fa fa-chevron-down fa-fw\"></i> Expand/Collapse All\n          </a>\n        </li>\n        <li>\n          <a download=\"items\" href=\"data:text/bibtex;base64,QGlucHJvY2VlZGluZ3N7cmFqX21lYXN1cmluZ18yMDIyLAoJdGl0bGUgPSB7TWVhc3VyaW5nIGZhaXJuZXNzIGluIHJhbmtlZCByZXN1bHRzOiBhbiBhbmFseXRpY2FsIGFuZCBlbXBpcmljYWwgY29tcGFyaXNvbn0sCgl1cmwgPSB7aHR0cHM6Ly9tZC5la3N0cmFuZG9tLm5ldC9wdWJzL2ZhaXItcmFua2luZ30sCglkb2kgPSB7MTAuMTE0NS8zNDc3NDk1LjM1MzIwMTh9LAoJYWJzdHJhY3QgPSB7SW5mb3JtYXRpb24gYWNjZXNzIHN5c3RlbXMsIHN1Y2ggYXMgc2VhcmNoIGFuZCByZWNvbW1lbmRlciBzeXN0ZW1zLCBvZnRlbiB1c2UgcmFua2VkIGxpc3RzIHRvIHByZXNlbnQgcmVzdWx0cyBiZWxpZXZlZCB0byBiZSByZWxldmFudCB0byB0aGUgdXNlcidzIGluZm9ybWF0aW9uIG5lZWQuIEV2YWx1YXRpbmcgdGhlc2UgbGlzdHMgZm9yIHRoZWlyIGZhaXJuZXNzIGFsb25nIHdpdGggb3RoZXIgdHJhZGl0aW9uYWwgbWV0cmljcyBwcm92aWRlcyBhIG1vcmUgY29tcGxldGUgdW5kZXJzdGFuZGluZyBvZiBhbiBpbmZvcm1hdGlvbiBhY2Nlc3Mgc3lzdGVtJ3MgYmVoYXZpb3IgYmV5b25kIGFjY3VyYWN5IG9yIHV0aWxpdHkgY29uc3RydWN0cy4gVG8gbWVhc3VyZSB0aGUgKHVuKWZhaXJuZXNzIG9mIHJhbmtpbmdzLCBwYXJ0aWN1bGFybHkgd2l0aCByZXNwZWN0IHRvIHRoZSBwcm90ZWN0ZWQgZ3JvdXAocykgb2YgcHJvZHVjZXJzIG9yIHByb3ZpZGVycywgc2V2ZXJhbCBtZXRyaWNzIGhhdmUgYmVlbiBwcm9wb3NlZCBpbiB0aGUgbGFzdCBzZXZlcmFsIHllYXJzLiBIb3dldmVyLCBhbiBlbXBpcmljYWwgYW5kIGNvbXBhcmF0aXZlIGFuYWx5c2VzIG9mIHRoZXNlIG1ldHJpY3Mgc2hvd2luZyB0aGUgYXBwbGljYWJpbGl0eSB0byBzcGVjaWZpYyBzY2VuYXJpbyBvciByZWFsIGRhdGEsIGNvbmNlcHR1YWwgc2ltaWxhcml0aWVzLCBhbmQgZGlmZmVyZW5jZXMgaXMgc3RpbGwgbGFja2luZy4KCldlIGFpbSB0byBicmlkZ2UgdGhlIGdhcCBiZXR3ZWVuIHRoZW9yZXRpY2FsIGFuZCBwcmFjdGljYWwgYXAtcGxpY2F0aW9uIG9mIHRoZXNlIG1ldHJpY3MuIEluIHRoaXMgcGFwZXIgd2UgZGVzY3JpYmUgc2V2ZXJhbCBmYWlyIHJhbmtpbmcgbWV0cmljcyBmcm9tIHRoZSBleGlzdGluZyBsaXRlcmF0dXJlIGluIGEgY29tbW9uIG5vdGF0aW9uLCBlbmFibGluZyBkaXJlY3QgY29tcGFyaXNvbiBvZiB0aGVpciBhcHByb2FjaGVzIGFuZCBhc3N1bXB0aW9ucywgYW5kIGVtcGlyaWNhbGx5IGNvbXBhcmUgdGhlbSBvbiB0aGUgc2FtZSBleHBlcmltZW50YWwgc2V0dXAgYW5kIGRhdGEgc2V0cyBpbiB0aGUgY29udGV4dCBvZiB0aHJlZSBpbmZvcm1hdGlvbiBhY2Nlc3MgdGFza3MuIFdlIGFsc28gcHJvdmlkZSBhIHNlbnNpdGl2aXR5IGFuYWx5c2lzIHRvIGFzc2VzcyB0aGUgaW1wYWN0IG9mIHRoZSBkZXNpZ24gY2hvaWNlcyBhbmQgcGFyYW1ldGVyIHNldHRpbmdzIHRoYXQgZ28gaW4gdG8gdGhlc2UgbWV0cmljcyBhbmQgcG9pbnQgdG8gYWRkaXRpb25hbCB3b3JrIG5lZWRlZCB0byBpbXByb3ZlIGZhaXJuZXNzIG1lYXN1cmVtZW50Ln0sCglib29rdGl0bGUgPSB7UHJvY2VlZGluZ3Mgb2YgdGhlIDQ1dGgge0ludGVybmF0aW9uYWx9IHtBQ019IHtTSUdJUn0ge0NvbmZlcmVuY2V9IG9uIHtSZXNlYXJjaH0gYW5kIHtEZXZlbG9wbWVudH0gaW4ge0luZm9ybWF0aW9ufSB7UmV0cmlldmFsfX0sCglwdWJsaXNoZXIgPSB7QUNNfSwKCWF1dGhvciA9IHtSYWosIEFtaWZhIGFuZCBFa3N0cmFuZCwgTWljaGFlbCBEfSwKCW1vbnRoID0ganVsLAoJeWVhciA9IHsyMDIyfSwKCXBhZ2VzID0gezcyNi0tNzM2fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tpaGVtZWxhbmR1X2NhbmRpZGF0ZV8yMDIzLAoJdGl0bGUgPSB7Q2FuZGlkYXRlIHNldCBzYW1wbGluZyBmb3IgZXZhbHVhdGluZyB0b3Ate059IHJlY29tbWVuZGF0aW9ufSwKCXVybCA9IHtodHRwOi8vYXJ4aXYub3JnL2Ficy8yMzA5LjExNzIzfSwKCWRvaSA9IHsxMC4xMTA5L1dJLUlBVDU5ODg4LjIwMjMuMDAwMTh9LAoJYWJzdHJhY3QgPSB7VGhlIHN0cmF0ZWd5IGZvciBzZWxlY3RpbmcgY2FuZGlkYXRlIHNldHMgLS0gdGhlIHNldCBvZiBpdGVtcyB0aGF0IHRoZSByZWNvbW1lbmRhdGlvbiBzeXN0ZW0gaXMgZXhwZWN0ZWQgdG8gcmFuayBmb3IgZWFjaCB1c2VyIC0tIGlzIGFuIGltcG9ydGFudCBkZWNpc2lvbiBpbiBjYXJyeWluZyBvdXQgYW4gb2ZmbGluZSB0b3AtXCROXCQgcmVjb21tZW5kZXIgc3lzdGVtIGV2YWx1YXRpb24uIFRoZSBzZXQgb2YgY2FuZGlkYXRlcyBpcyBjb21wb3NlZCBvZiB0aGUgdW5pb24gb2YgdGhlIHVzZXIncyB0ZXN0IGl0ZW1zIGFuZCBhbiBhcmJpdHJhcnkgbnVtYmVyIG9mIG5vbi1yZWxldmFudCBpdGVtcyB0aGF0IHdlIHJlZmVyIHRvIGFzIGRlY295cy4gUHJldmlvdXMgc3R1ZGllcyBoYXZlIGFpbWVkIHRvIHVuZGVyc3RhbmQgdGhlIGVmZmVjdCBvZiBkaWZmZXJlbnQgY2FuZGlkYXRlIHNldCBzaXplcyBhbmQgc2VsZWN0aW9uIHN0cmF0ZWdpZXMgb24gZXZhbHVhdGlvbi4gSW4gdGhpcyBwYXBlciwgd2UgZXh0ZW5kIHRoaXMga25vd2xlZGdlIGJ5IHN0dWR5aW5nIHRoZSBzcGVjaWZpYyBpbnRlcmFjdGlvbiBvZiBjYW5kaWRhdGUgc2V0IHNlbGVjdGlvbiBzdHJhdGVnaWVzIHdpdGggcG9wdWxhcml0eSBiaWFzLCBhbmQgdXNlIHNpbXVsYXRpb24gdG8gYXNzZXNzIHdoZXRoZXIgc2FtcGxlZCBjYW5kaWRhdGUgc2V0cyByZXN1bHQgaW4gbWV0cmljIGVzdGltYXRlcyB0aGF0IGFyZSBsZXNzIGJpYXNlZCB3aXRoIHJlc3BlY3QgdG8gdGhlIHRydWUgbWV0cmljIHZhbHVlcyB1bmRlciBjb21wbGV0ZSBkYXRhIHRoYXQgaXMgdHlwaWNhbGx5IHVuYXZhaWxhYmxlIGluIG9yZGluYXJ5IGV4cGVyaW1lbnRzLn0sCgl1cmxkYXRlID0gezIwMjMtMTEtMDh9LAoJYm9va3RpdGxlID0ge1Byb2NlZWRpbmdzIG9mIHRoZSAyMm5kIHtJRUVFfS97V0lDfSBpbnRlcm5hdGlvbmFsIGNvbmZlcmVuY2Ugb24gd2ViIGludGVsbGlnZW5jZSBhbmQgaW50ZWxsaWdlbnQgYWdlbnQgdGVjaG5vbG9neX0sCglhdXRob3IgPSB7SWhlbWVsYW5kdSwgTmdvemkgYW5kIEVrc3RyYW5kLCBNaWNoYWVsIEQufSwKCW1vbnRoID0gb2N0LAoJeWVhciA9IHsyMDIzfSwKCW5vdGUgPSB7YXJYaXY6MjMwOS4xMTcyMyBbY3NdfSwKCWtleXdvcmRzID0ge0NvbXB1dGVyIFNjaWVuY2UgLSBJbmZvcm1hdGlvbiBSZXRyaWV2YWx9LAoJcGFnZXMgPSB7ODgtLTk0fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tkaWF6X2V2YWx1YXRpbmdfMjAyMCwKCXNlcmllcyA9IHt7Q0lLTX0gJzIwfSwKCXRpdGxlID0ge0V2YWx1YXRpbmcge1N0b2NoYXN0aWN9IHtSYW5raW5nc30gd2l0aCB7RXhwZWN0ZWR9IHtFeHBvc3VyZX19LAoJdXJsID0ge2h0dHA6Ly9hcnhpdi5vcmcvYWJzLzIwMDQuMTMxNTd9LAoJZG9pID0gezEwLjExNDUvMzM0MDUzMS4zNDExOTYyfSwKCWFic3RyYWN0ID0ge1dlIGludHJvZHVjZSB0aGUgY29uY2VwdCBvZiBleHBlY3RlZCBleHBvc3VyZSBhcyB0aGUgYXZlcmFnZSBhdHRlbnRpb24KcmFua2VkIGl0ZW1zIHJlY2VpdmUgZnJvbSB1c2VycyBvdmVyIHJlcGVhdGVkIHNhbXBsZXMgb2YgdGhlIHNhbWUgcXVlcnkuCkZ1cnRoZXJtb3JlLCB3ZSBhZHZvY2F0ZSBmb3IgdGhlIGFkb3B0aW9uIG9mIHRoZSBwcmluY2lwbGUgb2YgZXF1YWwKZXhwZWN0ZWQgZXhwb3N1cmU6IGdpdmVuIGEgZml4ZWQgaW5mb3JtYXRpb24gbmVlZCwgbm8gaXRlbSByZWNlaXZlIG1vcmUgb3IKbGVzcyBleHBlY3RlZCBleHBvc3VyZSBjb21wYXJlZCB0byBhbnkgb3RoZXIgaXRlbSBvZiB0aGUgc2FtZSByZWxldmFuY2UKZ3JhZGUuIFdlIGFyZ3VlIHRoYXQgdGhpcyBwcmluY2lwbGUgaXMgZGVzaXJhYmxlIGZvciBtYW55IHJldHJpZXZhbApvYmplY3RpdmVzIGFuZCBzY2VuYXJpb3MsIGluY2x1ZGluZyB0b3BpY2FsIGRpdmVyc2l0eSBhbmQgZmFpciByYW5raW5nLgpMZXZlcmFnaW5nIHVzZXIgbW9kZWxzIGZyb20gZXhpc3RpbmcgcmV0cmlldmFsIG1ldHJpY3MsIHdlIHByb3Bvc2UgYQpnZW5lcmFsIGV2YWx1YXRpb24gbWV0aG9kb2xvZ3kgYmFzZWQgb24gZXhwZWN0ZWQgZXhwb3N1cmUgYW5kIGRyYXcKY29ubmVjdGlvbnMgdG8gcmVsYXRlZCBtZXRyaWNzIGluIGluZm9ybWF0aW9uIHJldHJpZXZhbCBldmFsdWF0aW9uLgpJbXBvcnRhbnRseSwgdGhpcyBtZXRob2RvbG9neSByZWxheGVzIGNsYXNzaWMgaW5mb3JtYXRpb24gcmV0cmlldmFsCmFzc3VtcHRpb25zLCBhbGxvd2luZyBhIHN5c3RlbSwgaW4gcmVzcG9uc2UgdG8gYSBxdWVyeSwgdG8gcHJvZHVjZSBhCmRpc3RyaWJ1dGlvbiBvdmVyIHJhbmtpbmdzIGluc3RlYWQgb2YgYSBzaW5nbGUgZml4ZWQgcmFua2luZy4gV2Ugc3R1ZHkgdGhlCmJlaGF2aW9yIG9mIHRoZSBleHBlY3RlZCBleHBvc3VyZSBtZXRyaWMgYW5kIHN0b2NoYXN0aWMgcmFua2VycyBhY3Jvc3MgYQp2YXJpZXR5IG9mIGluZm9ybWF0aW9uIGFjY2VzcyBjb25kaXRpb25zLCBpbmNsdWRpbmcgYWQgaG9jIHJldHJpZXZhbCBhbmQKcmVjb21tZW5kYXRpb24uIFdlIGJlbGlldmUgdGhhdCBtZWFzdXJpbmcgYW5kIG9wdGltaXppbmcgZXhwZWN0ZWQgZXhwb3N1cmUKbWV0cmljcyB1c2luZyByYW5kb21pemF0aW9uIG9wZW5zIGEgbmV3IGFyZWEgZm9yIHJldHJpZXZhbCBhbGdvcml0aG0KZGV2ZWxvcG1lbnQgYW5kIHByb2dyZXNzLn0sCglib29rdGl0bGUgPSB7UHJvY2VlZGluZ3Mgb2YgdGhlIDI5dGgge0FDTX0ge0ludGVybmF0aW9uYWx9IHtDb25mZXJlbmNlfSBvbiB7SW5mb3JtYXRpb259IGFuZCB7S25vd2xlZGdlfSB7TWFuYWdlbWVudH19LAoJcHVibGlzaGVyID0ge0FDTX0sCglhdXRob3IgPSB7RGlheiwgRmVybmFuZG8gYW5kIE1pdHJhLCBCaGFza2FyIGFuZCBFa3N0cmFuZCwgTWljaGFlbCBEIGFuZCBCaWVnYSwgQXNpYSBKIGFuZCBDYXJ0ZXJldHRlLCBCZW59LAoJbW9udGggPSBvY3QsCgl5ZWFyID0gezIwMjB9LAp9CgoKCkBpbnByb2NlZWRpbmdze3Jhal90b3dhcmRzXzIwMjQsCglzZXJpZXMgPSB7e0xOQ1N9fSwKCXRpdGxlID0ge1Rvd2FyZHMgb3B0aW1pemluZyByYW5raW5nIGluIGdyaWQtbGF5b3V0IGZvciBwcm92aWRlci1zaWRlIGZhaXJuZXNzfSwKCXZvbHVtZSA9IHsxNDYxMn0sCgljb3B5cmlnaHQgPSB7QWxsIHJpZ2h0cyByZXNlcnZlZH0sCgl1cmwgPSB7aHR0cHM6Ly9tZC5la3N0cmFuZG9tLm5ldC9wdWJzL2VjaXItZmFpci1ncmlkc30sCglkb2kgPSB7MTAuMTAwNy85NzgtMy0wMzEtNTYwNjktOV83fSwKCWFic3RyYWN0ID0ge0luZm9ybWF0aW9uIGFjY2VzcyBzeXN0ZW1zLCBzdWNoIGFzIHNlYXJjaCBlbmdpbmVzIGFuZCByZWNvbW1lbmRlciBzeXN0ZW1zLCBvcmRlciBhbmQgcG9zaXRpb24gcmVzdWx0cyBiYXNlZCBvbiB0aGVpciBlc3RpbWF0ZWQgcmVsZXZhbmNlLiBUaGVzZSByZXN1bHRzIGFyZSB0aGVuIGV2YWx1YXRlZCBmb3IgYSByYW5nZSBvZiBjb25jZXJucywgaW5jbHVkaW5nIHByb3ZpZGVyLXNpZGUgZmFpcm5lc3M6IHdoZXRoZXIgZXhwb3N1cmUgdG8gdXNlcnMgaXMgZmFpcmx5IGRpc3RyaWJ1dGVkIGFtb25nIGl0ZW1zIGFuZCB0aGUgcGVvcGxlIHdobyBjcmVhdGVkIHRoZW0uIFNldmVyYWwgZmFpcm5lc3MtYXdhcmUgcmFua2luZyBhbmQgcmUtcmFua2luZyB0ZWNobmlxdWVzIGhhdmUgYmVlbiBwcm9wb3NlZCB0byBlbnN1cmUgZmFpciBleHBvc3VyZSBmb3IgcHJvdmlkZXJzLCBidXQgdGhpcyB3b3JrIGZvY3VzZXMgYWxtb3N0IGV4Y2x1c2l2ZWx5IG9uIGxpbmVhciBsYXlvdXRzIGluIHdoaWNoIGl0ZW1zIGFyZSBkaXNwbGF5ZWQgaW4gc2luZ2xlIHJhbmtlZCBsaXN0LiBNYW55IHdpZGVseS11c2VkIHN5c3RlbXMgdXNlIG90aGVyIGxheW91dHMsIHN1Y2ggYXMgdGhlIGdyaWQgdmlld3MgY29tbW9uIGluIHN0cmVhbWluZyBwbGF0Zm9ybXMsIGltYWdlIHNlYXJjaCwgYW5kIG90aGVyIGFwcGxpY2F0aW9ucy4gUHJvdmlkaW5nIGZhaXIgZXhwb3N1cmUgdG8gcHJvdmlkZXJzIGluIHN1Y2ggbGF5b3V0cyBpcyBub3Qgd2VsbC1zdHVkaWVkLiBXZSBzZWVrIHRvIGZpbGwgdGhpcyBnYXAgYnkgcHJvdmlkaW5nIGEgZ3JpZC1hd2FyZSByZS1yYW5raW5nIGFsZ29yaXRobSB0byBvcHRpbWl6ZSBsYXlvdXRzIGZvciBwcm92aWRlci1zaWRlIGZhaXJuZXNzIGJ5IGFkYXB0aW5nIGV4aXN0aW5nIHJlLXJhbmtpbmcgdGVjaG5pcXVlcyB0byBncmlkLWF3YXJlIGJyb3dzaW5nIG1vZGVscywgYW5kIGFuIGFuYWx5c2lzIG9mIHRoZSBlZmZlY3Qgb2YgZ3JpZC1zcGVjaWZpYyBmYWN0b3JzIHN1Y2ggYXMgZGV2aWNlIHNpemUgb24gdGhlIHJlc3VsdGluZyBmYWlybmVzcyBvcHRpbWl6YXRpb24ufSwKCWxhbmd1YWdlID0ge2VufSwKCXVybGRhdGUgPSB7MjAyNC0wMS0wNH0sCglib29rdGl0bGUgPSB7QWR2YW5jZXMgaW4ge0luZm9ybWF0aW9ufSB7UmV0cmlldmFsfX0sCglwdWJsaXNoZXIgPSB7U3ByaW5nZXJ9LAoJYXV0aG9yID0ge1JhaiwgQW1pZmEgYW5kIEVrc3RyYW5kLCBNaWNoYWVsIEQufSwKCW1vbnRoID0gbWFyLAoJeWVhciA9IHsyMDI0fSwKCXBhZ2VzID0gezkwLS0xMDV9LAp9CgoKCkBpbnByb2NlZWRpbmdze2loZW1lbGFuZHVfbXVsdGlwbGVfMjAyNCwKCXNlcmllcyA9IHt7TE5DU319LAoJdGl0bGUgPSB7TXVsdGlwbGUgdGVzdGluZyBmb3Ige0lSfSBhbmQgcmVjb21tZW5kYXRpb24gc3lzdGVtIGV4cGVyaW1lbnRzfSwKCXZvbHVtZSA9IHsxNDYxMH0sCgl1cmwgPSB7aHR0cHM6Ly9tZC5la3N0cmFuZG9tLm5ldC9wdWJzL2VjaXItbWNwfSwKCWRvaSA9IHsxMC4xMDA3Lzk3OC0zLTAzMS01NjA2My03XzM3fSwKCWFic3RyYWN0ID0ge1doaWxlIHRoZXJlIGhhcyBiZWVuIHNpZ25pZmljYW50IHJlc2VhcmNoIG9uIHN0YXRpc3RpY2FsIHRlY2huaXF1ZXMgZm9yIGNvbXBhcmluZyB0d28gaW5mb3JtYXRpb24gcmV0cmlldmFsIChJUikgc3lzdGVtcywgbWFueSBJUiBleHBlcmltZW50cyB0ZXN0IG1vcmUgdGhhbiB0d28gc3lzdGVtcy4gVGhpcyBjYW4gbGVhZCB0byBpbmZsYXRlZCBmYWxzZSBkaXNjb3ZlcmllcyBkdWUgdG8gdGhlIG11bHRpcGxlLWNvbXBhcmlzb24gcHJvYmxlbSAoTUNQKS4gQSBmZXcgSVIgc3R1ZGllcyBoYXZlIGludmVzdGlnYXRlZCBtdWx0aXBsZSBjb21wYXJpc29uIHByb2NlZHVyZXM7IHRoZXNlIHN0dWRpZXMgbW9zdGx5IHVzZSBUUkVDIGRhdGEgYW5kIGNvbnRyb2wgdGhlIGZhbWlseXdpc2UgZXJyb3IgcmF0ZS4gSW4gdGhpcyBzdHVkeSwgd2UgZXh0ZW5kIHRoZWlyIGludmVzdGlnYXRpb24gdG8gaW5jbHVkZSByZWNvbW1lbmRhdGlvbiBzeXN0ZW0gZXZhbHVhdGlvbiBkYXRhIGFzIHdlbGwgYXMgbXVsdGlwbGUgY29tcGFyaXNvbiBwcm9jZWR1cmVzIHRoYXQgY29udHJvbHMgZm9yIEZhbHNlIERpc2NvdmVyeSBSYXRlIChGRFIpLn0sCglsYW5ndWFnZSA9IHtlbn0sCgl1cmxkYXRlID0gezIwMjQtMDEtMDR9LAoJYm9va3RpdGxlID0ge1Byb2NlZWRpbmdzIG9mIHRoZSA0NnRoIHtFdXJvcGVhbn0ge0NvbmZlcmVuY2V9IG9uIHtJbmZvcm1hdGlvbn0ge1JldHJpZXZhbH19LAoJcHVibGlzaGVyID0ge1NwcmluZ2VyfSwKCWF1dGhvciA9IHtJaGVtZWxhbmR1LCBOZ296aSBhbmQgRWtzdHJhbmQsIE1pY2hhZWwgRC59LAoJbW9udGggPSBtYXIsCgl5ZWFyID0gezIwMjR9LAoJcGFnZXMgPSB7NDQ5LS00NTd9LAp9CgoKCkBpbnByb2NlZWRpbmdze3dlZ21ldGhfcmV2ZWFsaW5nXzIwMjQsCgl0aXRsZSA9IHtSZXZlYWxpbmcgdGhlwqB7SGlkZGVufSB7SW1wYWN0fSBvZsKge1RvcH0te059IHtNZXRyaWNzfSBvbsKge09wdGltaXphdGlvbn0gaW7CoHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN9fSwKCWlzYm4gPSB7OTc4LTMtMDMxLTU2MDI3LTl9LAoJZG9pID0gezEwLjEwMDcvOTc4LTMtMDMxLTU2MDI3LTlfOX0sCglhYnN0cmFjdCA9IHtUaGUgaHlwZXJwYXJhbWV0ZXJzIG9mIHJlY29tbWVuZGVyIHN5c3RlbXMgZm9yIHRvcC1uIHByZWRpY3Rpb25zIGFyZSB0eXBpY2FsbHkgb3B0aW1pemVkIHRvIGVuaGFuY2UgdGhlIHByZWRpY3RpdmUgcGVyZm9ybWFuY2Ugb2YgYWxnb3JpdGhtcy4gVGhlcmVieSwgdGhlIG9wdGltaXphdGlvbiBhbGdvcml0aG0sIGUuZy4sIGdyaWQgc2VhcmNoIG9yIHJhbmRvbSBzZWFyY2gsIHNlYXJjaGVzIGZvciB0aGUgYmVzdCBoeXBlcnBhcmFtZXRlciBjb25maWd1cmF0aW9uIGFjY29yZGluZyB0byBhbiBvcHRpbWl6YXRpb24tdGFyZ2V0IG1ldHJpYywgbGlrZSBuRENHIG9yIFByZWNpc2lvbi4gSW4gY29udHJhc3QsIHRoZSBvcHRpbWl6ZWQgYWxnb3JpdGhtLCBlLmcuLCBBbHRlcm5hdGluZyBMZWFzdCBTcXVhcmVzIE1hdHJpeCBGYWN0b3JpemF0aW9uIG9yIEJheWVzaWFuIFBlcnNvbmFsaXplZCBSYW5raW5nLCBpbnRlcm5hbGx5IG9wdGltaXplcyBhIGRpZmZlcmVudCBsb3NzIGZ1bmN0aW9uIGR1cmluZyB0cmFpbmluZywgbGlrZSBzcXVhcmVkIGVycm9yIG9yIGNyb3NzLWVudHJvcHkuIFRvIHRhY2tsZSB0aGlzIGRpc2NyZXBhbmN5LCByZWNlbnQgd29yayBmb2N1c2VkIG9uIGdlbmVyYXRpbmcgbG9zcyBmdW5jdGlvbnMgYmV0dGVyIHN1aXRlZCBmb3IgcmVjb21tZW5kZXIgc3lzdGVtcy4gWWV0LCB3aGVuIGV2YWx1YXRpbmcgYW4gYWxnb3JpdGhtIHVzaW5nIGEgdG9wLW4gbWV0cmljIGR1cmluZyBvcHRpbWl6YXRpb24sIGFub3RoZXIgZGlzY3JlcGFuY3kgYmV0d2VlbiB0aGUgb3B0aW1pemF0aW9uLXRhcmdldCBtZXRyaWMgYW5kIHRoZSB0cmFpbmluZyBsb3NzIGhhcyBzbyBmYXIgYmVlbiBpZ25vcmVkLiBEdXJpbmcgb3B0aW1pemF0aW9uLCB0aGUgdG9wLW4gaXRlbXMgYXJlIHNlbGVjdGVkIGZvciBjb21wdXRpbmcgYSB0b3AtbiBtZXRyaWM7IGlnbm9yaW5nIHRoYXQgdGhlIHRvcC1uIGl0ZW1zIGFyZSBzZWxlY3RlZCBmcm9tIHRoZSByZWNvbW1lbmRhdGlvbnMgb2YgYSBtb2RlbCB0cmFpbmVkIHdpdGggYW4gZW50aXJlbHkgZGlmZmVyZW50IGxvc3MgZnVuY3Rpb24uIEl0ZW0gcmVjb21tZW5kYXRpb25zIHN1aXRhYmxlIGZvciBvcHRpbWl6YXRpb24tdGFyZ2V0IG1ldHJpY3MgY291bGQgYmUgb3V0c2lkZSB0aGUgdG9wLW4gcmVjb21tZW5kZWQgaXRlbXM7IGhpZGRlbmx5IGltcGFjdGluZyB0aGUgb3B0aW1pemF0aW9uIHBlcmZvcm1hbmNlLiBUaGVyZWZvcmUsIHdlIHdlcmUgbW90aXZhdGVkIHRvIGFuYWx5emUgd2hldGhlciB0aGUgdG9wLW4gaXRlbXMgYXJlIG9wdGltYWwgZm9yIG9wdGltaXphdGlvbi10YXJnZXQgdG9wLW4gbWV0cmljcy4gSW4gcHVyc3VpdCBvZiBhbiBhbnN3ZXIsIHdlIGV4aGF1c3RpdmVseSBldmFsdWF0ZSB0aGUgcHJlZGljdGl2ZSBwZXJmb3JtYW5jZSBvZiAyNTAgc2VsZWN0aW9uIHN0cmF0ZWdpZXMgYmVzaWRlcyBzZWxlY3RpbmcgdGhlIHRvcC1uLiBXZSBleHRlbnNpdmVseSBldmFsdWF0ZSBlYWNoIHNlbGVjdGlvbiBzdHJhdGVneSBvdmVyIHR3ZWx2ZSBpbXBsaWNpdCBmZWVkYmFjayBhbmQgZWlnaHQgZXhwbGljaXQgZmVlZGJhY2sgZGF0YSBzZXRzIHdpdGggZWxldmVuIHJlY29tbWVuZGVyIHN5c3RlbXMgYWxnb3JpdGhtcy4gT3VyIHJlc3VsdHMgc2hvdyB0aGF0IHRoZXJlIGV4aXN0IHNlbGVjdGlvbiBzdHJhdGVnaWVzIG90aGVyIHRoYW4gdG9wLW4gdGhhdCBpbmNyZWFzZSBwcmVkaWN0aXZlIHBlcmZvcm1hbmNlIGZvciB2YXJpb3VzIGFsZ29yaXRobXMgYW5kIHJlY29tbWVuZGF0aW9uIGRvbWFpbnMuIEhvd2V2ZXIsIHRoZSBwZXJmb3JtYW5jZSBvZiB0aGUgdG9wIFwkXCR7XHRleHRiYWNrc2xhc2h9c2ltIDQze1x0ZXh0YmFja3NsYXNofVwlXCRcJOKIvDQzXCVvZiBzZWxlY3Rpb24gc3RyYXRlZ2llcyBpcyBub3Qgc2lnbmlmaWNhbnRseSBkaWZmZXJlbnQuIFdlIGRpc2N1c3MgdGhlIGltcGFjdCBvZiBvdXIgZmluZGluZ3Mgb24gb3B0aW1pemF0aW9uIGFuZCByZS1yYW5raW5nIGluIHJlY29tbWVuZGVyIHN5c3RlbXMgYW5kIGZlYXNpYmxlIHNvbHV0aW9ucy4gVGhlIGltcGxlbWVudGF0aW9uIG9mIG91ciBzdHVkeSBpcyBwdWJsaWNseSBhdmFpbGFibGUufSwKCWxhbmd1YWdlID0ge2VufSwKCWJvb2t0aXRsZSA9IHtBZHZhbmNlcyBpbiB7SW5mb3JtYXRpb259IHtSZXRyaWV2YWx9fSwKCXB1Ymxpc2hlciA9IHtTcHJpbmdlciBOYXR1cmUgU3dpdHplcmxhbmR9LAoJYXV0aG9yID0ge1dlZ21ldGgsIEx1a2FzIGFuZCBWZW50ZSwgVG9iaWFzIGFuZCBQdXJ1Y2tlciwgTGVubmFydH0sCgllZGl0b3IgPSB7R29oYXJpYW4sIE5hemxpIGFuZCBUb25lbGxvdHRvLCBOaWNvbGEgYW5kIEhlLCBZdWxhbiBhbmQgTGlwYW5pLCBBbGRvIGFuZCBNY0RvbmFsZCwgR3JhaGFtIGFuZCBNYWNkb25hbGQsIENyYWlnIGFuZCBPdW5pcywgSWFkaH0sCgl5ZWFyID0gezIwMjR9LAoJcGFnZXMgPSB7MTQwLS0xNTZ9LAp9CgoKCkBpbnByb2NlZWRpbmdze3BhdGhha19lbXBpcmljYWxfMjAyNCwKCXNlcmllcyA9IHt7TE5DU319LAoJdGl0bGUgPSB7QW4ge0VtcGlyaWNhbH0ge0FuYWx5c2lzfSBvZsKge0ludGVydmVudGlvbn0ge1N0cmF0ZWdpZXN94oCZIHtFZmZlY3RpdmVuZXNzfSBmb3LCoHtDb3VudGVyaW5nfSB7TWlzaW5mb3JtYXRpb259IHtBbXBsaWZpY2F0aW9ufSBiecKge1JlY29tbWVuZGF0aW9ufSB7QWxnb3JpdGhtc319LAoJdm9sdW1lID0gezE0NjExfSwKCWlzYm4gPSB7OTc4LTMtMDMxLTU2MDY2LTh9LAoJZG9pID0gezEwLjEwMDcvOTc4LTMtMDMxLTU2MDY2LThfMjN9LAoJYWJzdHJhY3QgPSB7U29jaWFsIG5ldHdvcmsgcGxhdGZvcm1zIGNvbm5lY3QgcGVvcGxlIHdvcmxkd2lkZSwgZmFjaWxpdGF0aW5nIGNvbW11bmljYXRpb24sIGluZm9ybWF0aW9uIHNoYXJpbmcsIGFuZCBwZXJzb25hbC9wcm9mZXNzaW9uYWwgbmV0d29ya2luZy4gVGhleSB1c2UgcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcyB0byBwZXJzb25hbGl6ZSBjb250ZW50IGFuZCBlbmhhbmNlIHVzZXIgZXhwZXJpZW5jZXMuIEhvd2V2ZXIsIHRoZXNlIGFsZ29yaXRobXMgY2FuIHVuaW50ZW50aW9uYWxseSBhbXBsaWZ5IG1pc2luZm9ybWF0aW9uIGJ5IHByaW9yaXRpemluZyBlbmdhZ2VtZW50IG92ZXIgYWNjdXJhY3kuIEZvciBpbnN0YW5jZSwgcmVjZW50IHdvcmtzIHN1Z2dlc3QgdGhhdCBwb3B1bGFyaXR5LWJhc2VkIGFuZCBuZXR3b3JrLWJhc2VkIHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobXMgY29udHJpYnV0ZSB0aGUgbW9zdCB0byBtaXNpbmZvcm1hdGlvbiBkaWZmdXNpb24uIEluIG91ciBzdHVkeSwgd2UgcHJlc2VudCBhbiBleHBsb3JhdGlvbiBvbiB0d28gVHdpdHRlciBkYXRhc2V0cyB0byB1bmRlcnN0YW5kIHRoZSBpbXBhY3Qgb2YgaW50ZXJ2ZW50aW9uIHRlY2huaXF1ZXMgb24gY29tYmF0aW5nIG1pc2luZm9ybWF0aW9uIGFtcGxpZmljYXRpb24gaW5pdGlhdGVkIGJ5IHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobXMuIFdlIHNpbXVsYXRlIHZhcmlvdXMgc2NlbmFyaW9zIGFuZCBldmFsdWF0ZSB0aGUgZWZmZWN0aXZlbmVzcyBvZiBpbnRlcnZlbnRpb24gc3RyYXRlZ2llcyBpbiBzb2NpYWwgc2NpZW5jZXMgc3VjaCBhcyBWaXJhbGl0eSBDaXJjdWl0IEJyZWFrZXJzIGFuZCBhY2N1cmFjeSBudWRnZXMuIE91ciBmaW5kaW5ncyBoaWdobGlnaHQgdGhhdCB0aGVzZSBpbnRlcnZlbnRpb24gc3RyYXRlZ2llcyBhcmUgZ2VuZXJhbGx5IHN1Y2Nlc3NmdWwgd2hlbiBhcHBsaWVkIG9uIHRvcCBvZiBjb2xsYWJvcmF0aXZlIGZpbHRlcmluZyBhbmQgY29udGVudC1iYXNlZCByZWNvbW1lbmRhdGlvbiBhbGdvcml0aG1zLCB3aGlsZSBoYXZpbmcgZGlmZmVyZW50IGxldmVscyBvZiBlZmZlY3RpdmVuZXNzIGRlcGVuZGluZyBvbiB0aGUgbnVtYmVyIG9mIHVzZXJzIGtlZW4gdG8gc3ByZWFkIGZha2UgbmV3cyBwcmVzZW50IGluIHRoZSBkYXRhc2V0Ln0sCglsYW5ndWFnZSA9IHtlbn0sCglib29rdGl0bGUgPSB7QWR2YW5jZXMgaW4ge0luZm9ybWF0aW9ufSB7UmV0cmlldmFsfX0sCglwdWJsaXNoZXIgPSB7U3ByaW5nZXIgTmF0dXJlIFN3aXR6ZXJsYW5kfSwKCWF1dGhvciA9IHtQYXRoYWssIFJveWFsIGFuZCBTcGV6emFubywgRnJhbmNlc2NhfSwKCWVkaXRvciA9IHtHb2hhcmlhbiwgTmF6bGkgYW5kIFRvbmVsbG90dG8sIE5pY29sYSBhbmQgSGUsIFl1bGFuIGFuZCBMaXBhbmksIEFsZG8gYW5kIE1jRG9uYWxkLCBHcmFoYW0gYW5kIE1hY2RvbmFsZCwgQ3JhaWcgYW5kIE91bmlzLCBJYWRofSwKCXllYXIgPSB7MjAyNH0sCglwYWdlcyA9IHsyODUtLTMwMX0sCn0KCgoKQGFydGljbGV7d2FuZ19tb2RlbGluZ18yMDIzLAoJdGl0bGUgPSB7TW9kZWxpbmcgdW5jZXJ0YWludHkgdG8gaW1wcm92ZSBwZXJzb25hbGl6ZWQgcmVjb21tZW5kYXRpb25zIHZpYSB7QmF5ZXNpYW59IGRlZXAgbGVhcm5pbmd9LAoJdm9sdW1lID0gezE2fSwKCWlzc24gPSB7MjM2NC00MTY4fSwKCXVybCA9IHtodHRwczovL2RvaS5vcmcvMTAuMTAwNy9zNDEwNjAtMDIwLTAwMjQxLTF9LAoJZG9pID0gezEwLjEwMDcvczQxMDYwLTAyMC0wMDI0MS0xfSwKCWFic3RyYWN0ID0ge01vZGVsaW5nIHVuY2VydGFpbnR5IGhhcyBiZWVuIGEgbWFqb3IgY2hhbGxlbmdlIGluIGRldmVsb3BpbmcgTWFjaGluZSBMZWFybmluZyBzb2x1dGlvbnMgdG8gc29sdmUgcmVhbCB3b3JsZCBwcm9ibGVtcyBpbiB2YXJpb3VzIGRvbWFpbnMuIEluIFJlY29tbWVuZGVyIFN5c3RlbXMsIGEgdHlwaWNhbCB1c2FnZSBvZiB1bmNlcnRhaW50eSBpcyB0byBiYWxhbmNlIGV4cGxvcmF0aW9uIGFuZCBleHBsb2l0YXRpb24sIHdoZXJlIHRoZSB1bmNlcnRhaW50eSBoZWxwcyB0byBndWlkZSB0aGUgc2VsZWN0aW9uIG9mIG5ldyBvcHRpb25zIGluIGV4cGxvcmF0aW9uLiBSZWNlbnQgYWR2YW5jZXMgaW4gY29tYmluaW5nIEJheWVzaWFuIG1ldGhvZHMgd2l0aCBkZWVwIGxlYXJuaW5nIGVuYWJsZSB1cyB0byBleHByZXNzIHVuY2VydGFpbiBzdGF0dXMgaW4gZGVlcCBsZWFybmluZyBtb2RlbHMuIEluIHRoaXMgcGFwZXIsIHdlIGludmVzdGlnYXRlIGFuIGFwcHJvYWNoIGJhc2VkIG9uIEJheWVzaWFuIGRlZXAgbGVhcm5pbmcgdG8gaW1wcm92ZSBwZXJzb25hbGl6ZWQgcmVjb21tZW5kYXRpb25zLiBXZSBmaXJzdCBidWlsZCBkZWVwIGxlYXJuaW5nIGFyY2hpdGVjdHVyZXMgdG8gbGVhcm4gdXNlZnVsIHJlcHJlc2VudGF0aW9uIG9mIHVzZXIgYW5kIGl0ZW0gaW5wdXRzIGZvciBwcmVkaWN0aW5nIHRoZWlyIGludGVyYWN0aW9ucy4gV2UgdGhlbiBleHBsb3JlIG11bHRpcGxlIGVtYmVkZGluZyBjb21wb25lbnRzIHRvIGFjY29tbW9kYXRlIGRpZmZlcmVudCB0eXBlcyBvZiB1c2VyIGFuZCBpdGVtIGlucHV0cy4gQmFzZWQgb24gQmF5ZXNpYW4gZGVlcCBsZWFybmluZyB0ZWNobmlxdWVzLCBhIGtleSBub3ZlbHR5IG9mIG91ciBhcHByb2FjaCBpcyB0byBjYXB0dXJlIHRoZSB1bmNlcnRhaW50eSBhc3NvY2lhdGVkIHdpdGggdGhlIG1vZGVsIG91dHB1dCBhbmQgZnVydGhlciB1dGlsaXplIGl0IHRvIGJvb3N0IGV4cGxvcmF0aW9uIGluIHRoZSBjb250ZXh0IG9mIFJlY29tbWVuZGVyIFN5c3RlbXMuIFdlIHRlc3QgdGhlIHByb3Bvc2VkIGFwcHJvYWNoIGluIGJvdGggYSBDb2xsYWJvcmF0aXZlIEZpbHRlcmluZyBhbmQgYSBzaW11bGF0ZWQgb25saW5lIHJlY29tbWVuZGF0aW9uIHNldHRpbmcuIEV4cGVyaW1lbnRhbCByZXN1bHRzIG9uIHB1YmxpY2x5IGF2YWlsYWJsZSBiZW5jaG1hcmtzIGRlbW9uc3RyYXRlIHRoZSBiZW5lZml0cyBvZiBvdXIgYXBwcm9hY2ggaW4gaW1wcm92aW5nIHRoZSByZWNvbW1lbmRhdGlvbiBwZXJmb3JtYW5jZS59LAoJbGFuZ3VhZ2UgPSB7ZW59LAoJbnVtYmVyID0gezJ9LAoJdXJsZGF0ZSA9IHsyMDI0LTAzLTE3fSwKCWpvdXJuYWwgPSB7SW50ZXJuYXRpb25hbCBKb3VybmFsIG9mIERhdGEgU2NpZW5jZSBhbmQgQW5hbHl0aWNzfSwKCWF1dGhvciA9IHtXYW5nLCBYaW4gYW5kIEthZMSxb8SfbHUsIFNlcmRhcn0sCgltb250aCA9IGF1ZywKCXllYXIgPSB7MjAyM30sCglwYWdlcyA9IHsxOTEtLTIwMX0sCn0KCgoKQGFydGljbGV7bG9wZXNfcmVjb21tZW5kYXRpb25zXzIwMjQsCgl0aXRsZSA9IHtSZWNvbW1lbmRhdGlvbnMgd2l0aCBtaW5pbXVtIGV4cG9zdXJlIGd1YXJhbnRlZXM6IGEgcG9zdC1wcm9jZXNzaW5nIGZyYW1ld29ya30sCgl2b2x1bWUgPSB7MjM2fSwKCWlzc24gPSB7MDk1Ny00MTc0fSwKCXNob3J0dGl0bGUgPSB7UmVjb21tZW5kYXRpb25zIHdpdGggbWluaW11bSBleHBvc3VyZSBndWFyYW50ZWVzfSwKCXVybCA9IHtodHRwczovL3d3dy5zY2llbmNlZGlyZWN0LmNvbS9zY2llbmNlL2FydGljbGUvcGlpL1MwOTU3NDE3NDIzMDE2NjY0fSwKCWRvaSA9IHsxMC4xMDE2L2ouZXN3YS4yMDIzLjEyMTE2NH0sCglhYnN0cmFjdCA9IHtSZWxldmFuY2UtYmFzZWQgcmFua2luZyBpcyBhIHBvcHVsYXIgaW5ncmVkaWVudCBpbiByZWNvbW1lbmRlcnMsIGJ1dCBpdCBmcmVxdWVudGx5IHN0cnVnZ2xlcyB0byBtZWV0IGZhaXJuZXNzIGNyaXRlcmlhIGJlY2F1c2Ugc29jaWFsIGFuZCBjdWx0dXJhbCBub3JtcyBtYXkgZmF2b3Igc29tZSBpdGVtIGdyb3VwcyBvdmVyIG90aGVycy4gRm9yIGluc3RhbmNlLCBzb21lIGl0ZW1zIG1pZ2h0IHJlY2VpdmUgbG93ZXIgcmF0aW5ncyBkdWUgdG8gc29tZSBzb3J0IG9mIGJpYXMgKGUuZy4gZ2VuZGVyIGJpYXMpLiBBIGZhaXIgcmFua2luZyBzaG91bGQgYmFsYW5jZSB0aGUgZXhwb3N1cmUgb2YgaXRlbXMgZnJvbSBhZHZhbnRhZ2VkIGFuZCBkaXNhZHZhbnRhZ2VkIGdyb3Vwcy4gVG8gdGhpcyBlbmQsIHdlIHByb3Bvc2UgYSBub3ZlbCBwb3N0LXByb2Nlc3NpbmcgZnJhbWV3b3JrIHRvIHByb2R1Y2UgZmFpciwgZXhwb3N1cmUtYXdhcmUgcmVjb21tZW5kYXRpb25zLiBPdXIgYXBwcm9hY2ggaXMgYmFzZWQgb24gYW4gaW50ZWdlciBsaW5lYXIgcHJvZ3JhbW1pbmcgbW9kZWwgbWF4aW1pemluZyB0aGUgZXhwZWN0ZWQgdXRpbGl0eSB3aGlsZSBzYXRpc2Z5aW5nIGEgbWluaW11bSBleHBvc3VyZSBjb25zdHJhaW50LiBUaGUgbW9kZWwgaGFzIGZld2VyIHZhcmlhYmxlcyB0aGFuIHByZXZpb3VzIHdvcmsgYW5kIHRodXMgY2FuIGJlIGRlcGxveWVkIHRvIGxhcmdlciBkYXRhc2V0cyBhbmQgYWxsb3dzIHRoZSBvcmdhbml6YXRpb24gdG8gZGVmaW5lIGEgbWluaW11bSBsZXZlbCBvZiBleHBvc3VyZSBmb3IgZ3JvdXBzIG9mIGl0ZW1zLiBXZSBjb25kdWN0IGFuIGV4dGVuc2l2ZSBlbXBpcmljYWwgZXZhbHVhdGlvbiBpbmRpY2F0aW5nIHRoYXQgb3VyIG5ldyBmcmFtZXdvcmsgY2FuIGluY3JlYXNlIHRoZSBleHBvc3VyZSBvZiBpdGVtcyBmcm9tIGRpc2FkdmFudGFnZWQgZ3JvdXBzIGF0IGEgc21hbGwgY29zdCBvZiByZWNvbW1lbmRhdGlvbiBhY2N1cmFjeS59LAoJdXJsZGF0ZSA9IHsyMDIzLTA5LTE5fSwKCWpvdXJuYWwgPSB7RXhwZXJ0IFN5c3RlbXMgd2l0aCBBcHBsaWNhdGlvbnN9LAoJYXV0aG9yID0ge0xvcGVzLCBSYW1vbiBhbmQgQWx2ZXMsIFJvZHJpZ28gYW5kIExlZGVudCwgQW50b2luZSBhbmQgU2FudG9zLCBSb2RyeWdvIEwuIFQuIGFuZCBLbG9mdCwgTWFyaXVzfSwKCW1vbnRoID0gZmViLAoJeWVhciA9IHsyMDI0fSwKCWtleXdvcmRzID0ge0V4cG9zdXJlLCBGYWlybmVzcywgSW50ZWdlciBsaW5lYXIgcHJvZ3JhbW1pbmcsIFJlY29tbWVuZGVyIHN5c3RlbXMsIHRvLXJlYWR9LAoJcGFnZXMgPSB7MTIxMTY0fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3t3ZWdtZXRoX2VmZmVjdF8yMDIzLAoJdGl0bGUgPSB7VGhlIGVmZmVjdCBvZiByYW5kb20gc2VlZHMgZm9yIGRhdGEgc3BsaXR0aW5nIG9uIHJlY29tbWVuZGF0aW9uIGFjY3VyYWN5fSwKCWFic3RyYWN0ID0ge1RoZSBldmFsdWF0aW9uIG9mIHJlY29tbWVuZGVyIHN5c3RlbSBhbGdvcml0aG1zIGRlcGVuZHMgb24gcmFuZG9tbmVzcywgZS5nLiwgZHVyaW5nIHJhbmRvbWx5IHNwbGl0dGluZyBkYXRhIGludG8gdHJhaW5pbmcgYW5kIHRlc3RpbmcgZGF0YS4gV2Ugc3VzcGVjdCB0aGF0IGZhaWxpbmcgdG8gYWNjb3VudCBmb3IgcmFuZG9tbmVzcyBpbiB0aGlzIHNjZW5hcmlvIG1heSBsZWFkIHRvIG1pc3JlcHJlc2VudGluZyB0aGUgcHJlZGljdGl2ZSBhY2N1cmFjeSBvZiByZWNvbW1lbmRhdGlvbiBhbGdvcml0aG1zLiBUbyB1bmRlcnN0YW5kIHRoZSBjb21tdW5pdHnigJlzIHZpZXcgb2YgdGhlIGltcG9ydGFuY2Ugb2YgcmFuZG9tbmVzcywgd2UgY29uZHVjdGVkIGEgcGFwZXIgc3R1ZHkgb24gMzkgZnVsbCBwYXBlcnMgcHVibGlzaGVkIGF0IHRoZSBBQ00gUmVjU3lzIDIwMjIgY29uZmVyZW5jZS4gV2UgZm91bmQgdGhhdCB0aGUgYXV0aG9ycyBvZiAyNiBwYXBlcnMgdXNlZCBzb21lIHZhcmlhdGlvbiBvZiBhIGhvbGRvdXQgc3BsaXQgdGhhdCByZXF1aXJlcyBhIHJhbmRvbSBzZWVkLiBIb3dldmVyLCBvbmx5IGZpdmUgcGFwZXJzIGV4cGxpY2l0bHkgcmVwZWF0ZWQgZXhwZXJpbWVudHMgYW5kIGF2ZXJhZ2VkIHRoZWlyIHJlc3VsdHMgb3ZlciBkaWZmZXJlbnQgcmFuZG9tIHNlZWRzLiBUaGlzIHBvdGVudGlhbGx5IHByb2JsZW1hdGljIHJlc2VhcmNoIHByYWN0aWNlIG1vdGl2YXRlZCB1cyB0byBhbmFseXplIHRoZSBlZmZlY3Qgb2YgZGF0YSBzcGxpdCByYW5kb20gc2VlZHMgb24gcmVjb21tZW5kYXRpb24gYWNjdXJhY3kuIFRoZXJlZm9yZSwgd2UgdHJhaW4gdGhyZWUgY29tbW9uIGFsZ29yaXRobXMgb24gbmluZSBwdWJsaWMgZGF0YSBzZXRzIHdpdGggMjAgZGF0YSBzcGxpdCByYW5kb20gc2VlZHMsIGV2YWx1YXRlIHRoZW0gb24gdHdvIHJhbmtpbmcgbWV0cmljcyB3aXRoIHRocmVlIGRpZmZlcmVudCByYW5raW5nIGN1dG9mZiB2YWx1ZXMg8J2RmCwgYW5kIGNvbXBhcmUgdGhlIHJlc3VsdHMuIEluIHRoZSBleHRyZW1lIGNhc2Ugd2l0aCDwnZGYID0gMSwgd2Ugc2hvdyB0aGF0IGRlcGVuZGluZyBvbiB0aGUgZGF0YSBzcGxpdCByYW5kb20gc2VlZCwgdGhlIGFjY3VyYWN5IHdpdGggdHJhZGl0aW9uYWwgcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcyBkZXZpYXRlcyBieSB1cCB0byDiiLw2LjNcJSBmcm9tIHRoZSBtZWFuIGFjY3VyYWN5IGFjaGlldmVkIG9uIHRoZSBkYXRhIHNldC4gSGVuY2UsIHdlIHNob3cgdGhhdCBhbiBhbGdvcml0aG0gbWF5IHNpZ25pZmljYW50bHkgb3Zlci0gb3IgdW5kZXItcGVyZm9ybSB3aGVuIG1hbGljaW91c2x5IG9yIG5lZ2xpZ2VudGx5IHNlbGVjdGluZyBhIHJhbmRvbSBzZWVkIGZvciBzcGxpdHRpbmcgdGhlIGRhdGEuIFRvIHNob3djYXNlIGEgbWl0aWdhdGlvbiBzdHJhdGVneSBhbmQgYmV0dGVyIHJlc2VhcmNoIHByYWN0aWNlLCB3ZSBjb21wYXJlIGhvbGRvdXQgdG8gY3Jvc3MtdmFsaWRhdGlvbiBhbmQgc2hvdyB0aGF0LCBhZ2FpbiwgZm9yIPCdkZggPSAxLCB0aGUgYWNjdXJhY3kgb2YgYWxnb3JpdGhtcyBldmFsdWF0ZWQgd2l0aCBjcm9zcy12YWxpZGF0aW9uIGRldmlhdGVzIG9ubHkgdXAgdG8g4oi8Mi4zXCUgZnJvbSB0aGUgbWVhbiBhY2N1cmFjeSBhY2hpZXZlZCBvbiB0aGUgZGF0YSBzZXQuIEZ1cnRoZXJtb3JlLCB3ZSBmb3VuZCB0aGF0IHRoZSBkZXZpYXRpb24gYmVjb21lcyBzbWFsbGVyIHRoZSBoaWdoZXIgdGhlIHZhbHVlIG9mIPCdkZggZm9yIGJvdGggaG9sZG91dCBhbmQgY3Jvc3MtdmFsaWRhdGlvbi59LAoJbGFuZ3VhZ2UgPSB7ZW59LAoJYm9va3RpdGxlID0ge1BlcnNwZWN0aXZlcyBvbiB0aGUge0V2YWx1YXRpb259IG9mIHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN9IHtXb3Jrc2hvcH0gKHtQRVJTUEVDVElWRVN9IDIwMjMpfSwKCWF1dGhvciA9IHtXZWdtZXRoLCBMdWthcyBhbmQgVmVudGUsIFRvYmlhcyBhbmQgUHVydWNrZXIsIExlbm5hcnQgYW5kIEJlZWwsIEpvZXJhbn0sCgltb250aCA9IHNlcCwKCXllYXIgPSB7MjAyM30sCglrZXl3b3JkcyA9IHt0by1yZWFkfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3t2ZW50ZV9pbnRyb2R1Y2luZ18yMDIzLAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCglzZXJpZXMgPSB7e1JlY1N5c30gJzIzfSwKCXRpdGxlID0ge0ludHJvZHVjaW5nIHtMZW5zS2l0fS17QXV0b30sIGFuIGV4cGVyaW1lbnRhbCBhdXRvbWF0ZWQgcmVjb21tZW5kZXIgc3lzdGVtICh7QXV0b1JlY1N5c30pIHRvb2xraXR9LAoJaXNibiA9IHs5Nzk4NDAwNzAyNDE5fSwKCXVybCA9IHtodHRwczovL2RsLmFjbS5vcmcvZG9pLzEwLjExNDUvMzYwNDkxNS4zNjEwNjU2fSwKCWRvaSA9IHsxMC4xMTQ1LzM2MDQ5MTUuMzYxMDY1Nn0sCglhYnN0cmFjdCA9IHtMZW5zS2l0IGlzIG9uZSBvZiB0aGUgZmlyc3QgYW5kIG1vc3QgcG9wdWxhciBSZWNvbW1lbmRlciBTeXN0ZW0gbGlicmFyaWVzLiBXaGlsZSBMZW5zS2l0IG9mZmVycyBhIHdpZGUgdmFyaWV0eSBvZiBmZWF0dXJlcywgaXQgZG9lcyBub3QgaW5jbHVkZSBhbnkgb3B0aW1pemF0aW9uIHN0cmF0ZWdpZXMgb3IgZ3VpZGVsaW5lcyBvbiBob3cgdG8gc2VsZWN0IGFuZCB0dW5lIExlbnNLaXQgYWxnb3JpdGhtcy4gTGVuc0tpdCBkZXZlbG9wZXJzIGhhdmUgdG8gbWFudWFsbHkgaW5jbHVkZSB0aGlyZC1wYXJ0eSBsaWJyYXJpZXMgaW50byB0aGVpciBleHBlcmltZW50YWwgc2V0dXAgb3IgaW1wbGVtZW50IG9wdGltaXphdGlvbiBzdHJhdGVnaWVzIGJ5IGhhbmQgdG8gb3B0aW1pemUgaHlwZXJwYXJhbWV0ZXJzLiBXZSBmb3VuZCB0aGF0IDYzLjZcJSAoMjEgb3V0IG9mIDMzKSBvZiBwYXBlcnMgdXNpbmcgTGVuc0tpdCBhbGdvcml0aG1zIGZvciB0aGVpciBleHBlcmltZW50cyBkaWQgbm90IHNlbGVjdCBhbGdvcml0aG1zIG9yIHR1bmUgaHlwZXJwYXJhbWV0ZXJzLiBOb24tb3B0aW1pemVkIG1vZGVscyByZXByZXNlbnQgcG9vciBiYXNlbGluZXMgYW5kIHByb2R1Y2UgbGVzcyBtZWFuaW5nZnVsIHJlc2VhcmNoIHJlc3VsdHMuIFRoaXMgZGVtbyBpbnRyb2R1Y2VzIExlbnNLaXQtQXV0by4gTGVuc0tpdC1BdXRvIGF1dG9tYXRlcyB0aGUgZW50aXJlIFJlY29tbWVuZGVyIFN5c3RlbSBwaXBlbGluZSBhbmQgZW5hYmxlcyBMZW5zS2l0IGRldmVsb3BlcnMgdG8gYXV0b21hdGljYWxseSBzZWxlY3QsIG9wdGltaXplLCBhbmQgZW5zZW1ibGUgTGVuc0tpdCBhbGdvcml0aG1zLn0sCgl1cmxkYXRlID0gezIwMjMtMDktMTh9LAoJYm9va3RpdGxlID0ge1Byb2NlZWRpbmdzIG9mIHRoZSAxN3RoIHtBQ019IHtDb25mZXJlbmNlfSBvbiB7UmVjb21tZW5kZXJ9IHtTeXN0ZW1zfX0sCglwdWJsaXNoZXIgPSB7QXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAoJYXV0aG9yID0ge1ZlbnRlLCBUb2JpYXMgYW5kIEVrc3RyYW5kLCBNaWNoYWVsIGFuZCBCZWVsLCBKb2VyYW59LAoJbW9udGggPSBzZXAsCgl5ZWFyID0gezIwMjN9LAoJa2V5d29yZHMgPSB7QWxnb3JpdGhtIFNlbGVjdGlvbiwgQXV0b1JlY1N5cywgQXV0b21hdGVkIFJlY29tbWVuZGVyIFN5c3RlbXMsIENBU0gsIEh5cGVycGFyYW1ldGVyIE9wdGltaXphdGlvbiwgUmVjb21tZW5kZXIgU3lzdGVtc30sCglwYWdlcyA9IHsxMjEyLS0xMjE2fSwKfQoKCgpAbWlzY3tsaV9taXRpZ2F0aW5nXzIwMjMsCgl0aXRsZSA9IHtNaXRpZ2F0aW5nIG1haW5zdHJlYW0gYmlhcyBpbiByZWNvbW1lbmRhdGlvbiB2aWEgY29zdC1zZW5zaXRpdmUgbGVhcm5pbmd9LAoJdXJsID0ge2h0dHA6Ly9hcnhpdi5vcmcvYWJzLzIzMDcuMTM2MzJ9LAoJZG9pID0gezEwLjExNDUvMzU3ODMzNy4zNjA1MTM0fSwKCWFic3RyYWN0ID0ge01haW5zdHJlYW0gYmlhcywgd2hlcmUgc29tZSB1c2VycyByZWNlaXZlIHBvb3IgcmVjb21tZW5kYXRpb25zIGJlY2F1c2UgdGhlaXIgcHJlZmVyZW5jZXMgYXJlIHVuY29tbW9uIG9yIHNpbXBseSBiZWNhdXNlIHRoZXkgYXJlIGxlc3MgYWN0aXZlLCBpcyBhbiBpbXBvcnRhbnQgYXNwZWN0IHRvIGNvbnNpZGVyIHJlZ2FyZGluZyBmYWlybmVzcyBpbiByZWNvbW1lbmRlciBzeXN0ZW1zLiBFeGlzdGluZyBtZXRob2RzIHRvIG1pdGlnYXRlIG1haW5zdHJlYW0gYmlhcyBkbyBub3QgZXhwbGljaXRseSBtb2RlbCB0aGUgaW1wb3J0YW5jZSBvZiB0aGVzZSBub24tbWFpbnN0cmVhbSB1c2VycyBvciwgd2hlbiB0aGV5IGRvLCBpdCBpcyBpbiBhIHdheSB0aGF0IGlzIG5vdCBuZWNlc3NhcmlseSBjb21wYXRpYmxlIHdpdGggdGhlIGRhdGEgYW5kIHJlY29tbWVuZGF0aW9uIG1vZGVsIGF0IGhhbmQuIEluIGNvbnRyYXN0LCB3ZSB1c2UgdGhlIHJlY29tbWVuZGF0aW9uIHV0aWxpdHkgYXMgYSBtb3JlIGdlbmVyaWMgYW5kIGltcGxpY2l0IHByb3h5IHRvIHF1YW50aWZ5IG1haW5zdHJlYW1uZXNzLCBhbmQgcHJvcG9zZSBhIHNpbXBsZSB1c2VyLXdlaWdodGluZyBhcHByb2FjaCB0byBpbmNvcnBvcmF0ZSBpdCBpbnRvIHRoZSB0cmFpbmluZyBwcm9jZXNzIHdoaWxlIHRha2luZyB0aGUgY29zdCBvZiBwb3RlbnRpYWwgcmVjb21tZW5kYXRpb24gZXJyb3JzIGludG8gYWNjb3VudC4gV2UgcHJvdmlkZSBleHRlbnNpdmUgZXhwZXJpbWVudGFsIHJlc3VsdHMgc2hvd2luZyB0aGF0IHF1YW50aWZ5aW5nIG1haW5zdHJlYW1uZXNzIHZpYSB1dGlsaXR5IGlzIGJldHRlciBhYmxlIGF0IGlkZW50aWZ5aW5nIG5vbi1tYWluc3RyZWFtIHVzZXJzLCBhbmQgdGhhdCB0aGV5IGFyZSBpbmRlZWQgYmV0dGVyIHNlcnZlZCB3aGVuIHRyYWluaW5nIHRoZSBtb2RlbCBpbiBhIGNvc3Qtc2Vuc2l0aXZlIHdheS4gVGhpcyBpcyBhY2hpZXZlZCB3aXRoIG5lZ2xpZ2libGUgb3Igbm8gbG9zcyBpbiBvdmVyYWxsIHJlY29tbWVuZGF0aW9uIGFjY3VyYWN5LCBtZWFuaW5nIHRoYXQgdGhlIG1vZGVscyBsZWFybiBhIGJldHRlciBiYWxhbmNlIGFjcm9zcyB1c2Vycy4gSW4gYWRkaXRpb24sIHdlIHNob3cgdGhhdCByZXNlYXJjaCBvZiB0aGlzIGtpbmQsIHdoaWNoIGV2YWx1YXRlcyByZWNvbW1lbmRhdGlvbiBxdWFsaXR5IGF0IHRoZSBpbmRpdmlkdWFsIHVzZXIgbGV2ZWwsIG1heSBub3QgYmUgcmVsaWFibGUgaWYgbm90IHVzaW5nIGVub3VnaCBpbnRlcmFjdGlvbnMgd2hlbiBhc3Nlc3NpbmcgbW9kZWwgcGVyZm9ybWFuY2UufSwKCXVybGRhdGUgPSB7MjAyMy0wNy0yOX0sCglhdXRob3IgPSB7TGksIFJvZ2VyIFpoZSBhbmQgVXJiYW5vLCBKdWxpw6FuIGFuZCBIYW5qYWxpYywgQWxhbn0sCgltb250aCA9IGp1bCwKCXllYXIgPSB7MjAyM30sCglub3RlID0ge2FyWGl2OjIzMDcuMTM2MzIgW2NzXX0sCglrZXl3b3JkcyA9IHtDb21wdXRlciBTY2llbmNlIC0gSW5mb3JtYXRpb24gUmV0cmlldmFsfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tpaGVtZWxhbmR1X2luZmVyZW5jZV8yMDIzLAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCglzZXJpZXMgPSB7e1NJR0lSfSAnMjN9LAoJdGl0bGUgPSB7SW5mZXJlbmNlIGF0IHNjYWxlOiBzaWduaWZpY2FuY2UgdGVzdGluZyBmb3IgbGFyZ2Ugc2VhcmNoIGFuZCByZWNvbW1lbmRhdGlvbiBleHBlcmltZW50c30sCgljb3B5cmlnaHQgPSB7QWxsIHJpZ2h0cyByZXNlcnZlZH0sCglpc2JuID0gezk3OC0xLTQ1MDMtOTQwOC02fSwKCXNob3J0dGl0bGUgPSB7SW5mZXJlbmNlIGF0IHNjYWxlfSwKCXVybCA9IHtodHRwczovL2RsLmFjbS5vcmcvZG9pLzEwLjExNDUvMzUzOTYxOC4zNTkyMDA0fSwKCWRvaSA9IHsxMC4xMTQ1LzM1Mzk2MTguMzU5MjAwNH0sCglhYnN0cmFjdCA9IHtBIG51bWJlciBvZiBpbmZvcm1hdGlvbiByZXRyaWV2YWwgc3R1ZGllcyBoYXZlIGJlZW4gZG9uZSB0byBhc3Nlc3Mgd2hpY2ggc3RhdGlzdGljYWwgdGVjaG5pcXVlcyBhcmUgYXBwcm9wcmlhdGUgZm9yIGNvbXBhcmluZyBzeXN0ZW1zLiBIb3dldmVyLCB0aGVzZSBzdHVkaWVzIGFyZSBmb2N1c2VkIG9uIFRSRUMtc3R5bGUgZXhwZXJpbWVudHMsIHdoaWNoIHR5cGljYWxseSBoYXZlIGZld2VyIHRoYW4gMTAwIHRvcGljcy4gVGhlcmUgaXMgbm8gc2ltaWxhciBsaW5lIG9mIHdvcmsgZm9yIGxhcmdlIHNlYXJjaCBhbmQgcmVjb21tZW5kYXRpb24gZXhwZXJpbWVudHM7IHN1Y2ggc3R1ZGllcyB0eXBpY2FsbHkgaGF2ZSB0aG91c2FuZHMgb2YgdG9waWNzIG9yIHVzZXJzIGFuZCBtdWNoIHNwYXJzZXIgcmVsZXZhbmNlIGp1ZGdlbWVudHMsIHNvIGl0IGlzIG5vdCBjbGVhciBpZiByZWNvbW1lbmRhdGlvbnMgZm9yIGFuYWx5emluZyB0cmFkaXRpb25hbCBUUkVDIGV4cGVyaW1lbnRzIGFwcGx5IHRvIHRoZXNlIHNldHRpbmdzLiBJbiB0aGlzIHBhcGVyLCB3ZSBlbXBpcmljYWxseSBzdHVkeSB0aGUgYmVoYXZpb3Igb2Ygc2lnbmlmaWNhbmNlIHRlc3RzIHdpdGggbGFyZ2Ugc2VhcmNoIGFuZCByZWNvbW1lbmRhdGlvbiBldmFsdWF0aW9uIGRhdGEuIE91ciByZXN1bHRzIHNob3cgdGhhdCB0aGUgV2lsY294b24gYW5kIFNpZ24gdGVzdHMgc2hvdyBzaWduaWZpY2FudGx5IGhpZ2hlciBUeXBlLTEgZXJyb3IgcmF0ZXMgZm9yIGxhcmdlIHNhbXBsZSBzaXplcyB0aGFuIHRoZSBib290c3RyYXAsIHJhbmRvbWl6YXRpb24gYW5kIHQtdGVzdHMsIHdoaWNoIHdlcmUgbW9yZSBjb25zaXN0ZW50IHdpdGggdGhlIGV4cGVjdGVkIGVycm9yIHJhdGUuIFdoaWxlIHRoZSBzdGF0aXN0aWNhbCB0ZXN0cyBkaXNwbGF5ZWQgZGlmZmVyZW5jZXMgaW4gdGhlaXIgcG93ZXIgZm9yIHNtYWxsZXIgc2FtcGxlIHNpemVzLCB0aGV5IHNob3dlZCBubyBkaWZmZXJlbmNlIGluIHRoZWlyIHBvd2VyIGZvciBsYXJnZSBzYW1wbGUgc2l6ZXMuIFdlIHJlY29tbWVuZCB0aGUgc2lnbiBhbmQgV2lsY294b24gdGVzdHMgc2hvdWxkIG5vdCBiZSB1c2VkIHRvIGFuYWx5emUgbGFyZ2Ugc2NhbGUgZXZhbHVhdGlvbiByZXN1bHRzLiBPdXIgcmVzdWx0IGRlbW9uc3RyYXRlIHRoYXQgd2l0aCBUb3AtTiByZWNvbW1lbmRhdGlvbiBhbmQgbGFyZ2Ugc2VhcmNoIGV2YWx1YXRpb24gZGF0YSwgbW9zdCB0ZXN0cyB3b3VsZCBoYXZlIGEgMTAwXCUgY2hhbmNlIG9mIGZpbmRpbmcgc3RhdGlzdGljYWxseSBzaWduaWZpY2FudCByZXN1bHRzLiBUaGVyZWZvcmUsIHRoZSBlZmZlY3Qgc2l6ZSBzaG91bGQgYmUgdXNlZCB0byBkZXRlcm1pbmUgcHJhY3RpY2FsIG9yIHNjaWVudGlmaWMgc2lnbmlmaWNhbmNlLn0sCgl1cmxkYXRlID0gezIwMjMtMDctMjN9LAoJYm9va3RpdGxlID0ge1Byb2NlZWRpbmdzIG9mIHRoZSA0NnRoIHtJbnRlcm5hdGlvbmFsfSB7QUNNfSB7U0lHSVJ9IHtDb25mZXJlbmNlfSBvbiB7UmVzZWFyY2h9IGFuZCB7RGV2ZWxvcG1lbnR9IGluIHtJbmZvcm1hdGlvbn0ge1JldHJpZXZhbH19LAoJcHVibGlzaGVyID0ge0Fzc29jaWF0aW9uIGZvciBDb21wdXRpbmcgTWFjaGluZXJ5fSwKCWF1dGhvciA9IHtJaGVtZWxhbmR1LCBOZ296aSBhbmQgRWtzdHJhbmQsIE1pY2hhZWwgRC59LAoJbW9udGggPSBqdWwsCgl5ZWFyID0gezIwMjN9LAoJa2V5d29yZHMgPSB7ZXZhbHVhdGlvbiwgc3RhdGlzdGljYWwgaW5mZXJlbmNlfSwKCXBhZ2VzID0gezIwODctLTIwOTF9LAp9CgoKCkBpbnByb2NlZWRpbmdze3Jhal9jb21wYXJpbmdfMjAyMCwKCXRpdGxlID0ge0NvbXBhcmluZyBmYWlyIHJhbmtpbmcgbWV0cmljc30sCgl1cmwgPSB7aHR0cDovL2FyeGl2Lm9yZy9hYnMvMjAwOS4wMTMxMX0sCglhYnN0cmFjdCA9IHtSYW5raW5nIGlzIGEgZnVuZGFtZW50YWwgYXNwZWN0IG9mIHJlY29tbWVuZGVyIHN5c3RlbXMuIEhvd2V2ZXIsIHJhbmtlZApvdXRwdXRzIGNhbiBiZSBzdXNjZXB0aWJsZSB0byB2YXJpb3VzIGJpYXNlczsgc29tZSBvZiB0aGVzZSBtYXkgY2F1c2UKZGlzYWR2YW50YWdlcyB0byBtZW1iZXJzIG9mIHByb3RlY3RlZCBncm91cHMuIFNldmVyYWwgbWV0cmljcyBoYXZlIGJlZW4KcHJvcG9zZWQgdG8gcXVhbnRpZnkgdGhlICh1bilmYWlybmVzcyBvZiByYW5raW5ncywgYnV0IHRoZXJlIGhhcyBub3QgYmVlbgp0byBkYXRlIGFueSBkaXJlY3QgY29tcGFyaXNvbiBvZiB0aGVzZSBtZXRyaWNzLiBUaGlzIGNvbXBsaWNhdGVzIGRlY2lkaW5nCndoYXQgZmFpcm5lc3MgbWV0cmljcyBhcmUgYXBwbGljYWJsZSBmb3Igc3BlY2lmaWMgc2NlbmFyaW9zLCBhbmQgYXNzZXNzaW5nCnRoZSBleHRlbnQgdG8gd2hpY2ggbWV0cmljcyBhZ3JlZSBvciBkaXNhZ3JlZS4gSW4gdGhpcyBwYXBlciwgd2UgZGVzY3JpYmUKc2V2ZXJhbCBmYWlyIHJhbmtpbmcgbWV0cmljcyBpbiBhIGNvbW1vbiBub3RhdGlvbiwgZW5hYmxpbmcgZGlyZWN0CmNvbXBhcmlzb24gb2YgdGhlaXIgYXBwcm9hY2hlcyBhbmQgYXNzdW1wdGlvbnMsIGFuZCBlbXBpcmljYWxseSBjb21wYXJlCnRoZW0gb24gdGhlIHNhbWUgZXhwZXJpbWVudGFsIHNldHVwIGFuZCBkYXRhIHNldC4gT3VyIHdvcmsgcHJvdmlkZXMgYQpkaXJlY3QgY29tcGFyYXRpdmUgYW5hbHlzaXMgaWRlbnRpZnlpbmcgc2ltaWxhcml0aWVzIGFuZCBkaWZmZXJlbmNlcyBvZgpmYWlyIHJhbmtpbmcgbWV0cmljcyBzZWxlY3RlZCBmb3Igb3VyIHdvcmsufSwKCWF1dGhvciA9IHtSYWosIEFtaWZhIGFuZCBXb29kLCBDb25ub3IgYW5kIE1vbnRvbHksIEFuYW5kYSBhbmQgRWtzdHJhbmQsIE1pY2hhZWwgRH0sCgltb250aCA9IHNlcCwKCXllYXIgPSB7MjAyMH0sCn0KCgoKQGFydGljbGV7ZWtzdHJhbmRfZXhwbG9yaW5nXzIwMjEsCgl0aXRsZSA9IHtFeHBsb3JpbmcgYXV0aG9yIGdlbmRlciBpbiBib29rIHJhdGluZyBhbmQgcmVjb21tZW5kYXRpb259LAoJdm9sdW1lID0gezMxfSwKCWlzc24gPSB7MDkyNC0xODY4fSwKCXVybCA9IHtodHRwczovL21kLmVrc3RyYW5kb20ubmV0L3B1YnMvYmFnLWV4dGVuZGVkfSwKCWRvaSA9IHsxMC4xMDA3L3MxMTI1Ny0wMjAtMDkyODQtMn0sCglhYnN0cmFjdCA9IHtDb2xsYWJvcmF0aXZlIGZpbHRlcmluZyBhbGdvcml0aG1zIGZpbmQgdXNlZnVsIHBhdHRlcm5zIGluIHJhdGluZyBhbmQgY29uc3VtcHRpb24gZGF0YSBhbmQgZXhwbG9pdCB0aGVzZSBwYXR0ZXJucyB0byBndWlkZSB1c2VycyB0byBnb29kIGl0ZW1zLiBNYW55IG9mIHRoZSBwYXR0ZXJucyBpbiByYXRpbmcgZGF0YXNldHMgcmVmbGVjdCBpbXBvcnRhbnQgcmVhbC13b3JsZCBkaWZmZXJlbmNlcyBiZXR3ZWVuIHRoZSB2YXJpb3VzIHVzZXJzIGFuZCBpdGVtcyBpbiB0aGUgZGF0YTsgb3RoZXIgcGF0dGVybnMgbWF5IGJlIGlycmVsZXZhbnQgb3IgcG9zc2libHkgdW5kZXNpcmFibGUgZm9yIHNvY2lhbCBvciBldGhpY2FsIHJlYXNvbnMsIHBhcnRpY3VsYXJseSBpZiB0aGV5IHJlZmxlY3QgdW5kZXNpcmVkIGRpc2NyaW1pbmF0aW9uLCBzdWNoIGFzIGRpc2NyaW1pbmF0aW9uIGluIHB1Ymxpc2hpbmcgb3IgcHVyY2hhc2luZyBhZ2FpbnN0IGF1dGhvcnMgd2hvIGFyZSB3b21lbiBvciBldGhuaWMgbWlub3JpdGllcy4gSW4gdGhpcyB3b3JrLCB3ZSBleGFtaW5lIHRoZSByZXNwb25zZSBvZiBjb2xsYWJvcmF0aXZlIGZpbHRlcmluZyByZWNvbW1lbmRlciBhbGdvcml0aG1zIHRvIHRoZSBkaXN0cmlidXRpb24gb2YgdGhlaXIgaW5wdXQgZGF0YSB3aXRoIHJlc3BlY3QgdG8gYSBkaW1lbnNpb24gb2Ygc29jaWFsIGNvbmNlcm4sIG5hbWVseSBjb250ZW50IGNyZWF0b3IgZ2VuZGVyLiBVc2luZyBwdWJsaWNseS1hdmFpbGFibGUgYm9vayByYXRpbmdzIGRhdGEsIHdlIG1lYXN1cmUgdGhlIGRpc3RyaWJ1dGlvbiBvZiB0aGUgZ2VuZGVycyBvZiB0aGUgYXV0aG9ycyBvZiBib29rcyBpbiB1c2VyIHJhdGluZyBwcm9maWxlcyBhbmQgcmVjb21tZW5kYXRpb24gbGlzdHMgcHJvZHVjZWQgZnJvbSB0aGlzIGRhdGEuIFdlIGZpbmQgdGhhdCBjb21tb24gY29sbGFib3JhdGl2ZSBmaWx0ZXJpbmcgYWxnb3JpdGhtcyBkaWZmZXIgaW4gdGhlIGdlbmRlciBkaXN0cmlidXRpb24gb2YgdGhlaXIgcmVjb21tZW5kYXRpb24gbGlzdHMsIGFuZCBpbiB0aGUgcmVsYXRpb25zaGlwIG9mIHRoYXQgb3V0cHV0IGRpc3RyaWJ1dGlvbiB0byB1c2VyIHByb2ZpbGUgZGlzdHJpYnV0aW9uLn0sCgludW1iZXIgPSB7M30sCgl1cmxkYXRlID0gezIwMjAtMDYtMDV9LAoJam91cm5hbCA9IHtVc2VyIE1vZGVsaW5nIGFuZCBVc2VyLUFkYXB0ZWQgSW50ZXJhY3Rpb259LAoJYXV0aG9yID0ge0Vrc3RyYW5kLCBNaWNoYWVsIEQgYW5kIEtsdXZlciwgRGFuaWVsfSwKCW1vbnRoID0ganVsLAoJeWVhciA9IHsyMDIxfSwKCXBhZ2VzID0gezM3Ny0tNDIwfSwKfQoKCgpAYXJ0aWNsZXtla3N0cmFuZF9kaXN0cmlidXRpb25hbGx5LWluZm9ybWVkXzIwMjMsCgl0aXRsZSA9IHtEaXN0cmlidXRpb25hbGx5LWluZm9ybWVkIHJlY29tbWVuZGVyIHN5c3RlbSBldmFsdWF0aW9ufSwKCWNvcHlyaWdodCA9IHtBbGwgcmlnaHRzIHJlc2VydmVkfSwKCXVybCA9IHtodHRwczovL2RsLmFjbS5vcmcvZG9pLzEwLjExNDUvMzYxMzQ1NX0sCglkb2kgPSB7MTAuMTE0NS8zNjEzNDU1fSwKCWFic3RyYWN0ID0ge0N1cnJlbnQgcHJhY3RpY2UgZm9yIGV2YWx1YXRpbmcgcmVjb21tZW5kZXIgc3lzdGVtcyB0eXBpY2FsbHkgZm9jdXNlcyBvbiBwb2ludCBlc3RpbWF0ZXMgb2YgdXNlci1vcmllbnRlZCBlZmZlY3RpdmVuZXNzIG1ldHJpY3Mgb3IgYnVzaW5lc3MgbWV0cmljcywgc29tZXRpbWVzIGNvbWJpbmVkIHdpdGggYWRkaXRpb25hbCBtZXRyaWNzIGZvciBjb25zaWRlcmF0aW9ucyBzdWNoIGFzIGRpdmVyc2l0eSBhbmQgbm92ZWx0eS4gSW4gdGhpcyBwYXBlciwgd2UgYXJndWUgZm9yIHRoZSBuZWVkIGZvciByZXNlYXJjaGVycyBhbmQgcHJhY3RpdGlvbmVycyB0byBhdHRlbmQgbW9yZSBjbG9zZWx5IHRvIHZhcmlvdXMgZGlzdHJpYnV0aW9ucyB0aGF0IGFyaXNlIGZyb20gYSByZWNvbW1lbmRlciBzeXN0ZW0gKG9yIG90aGVyIGluZm9ybWF0aW9uIGFjY2VzcyBzeXN0ZW0pIGFuZCB0aGUgc291cmNlcyBvZiB1bmNlcnRhaW50eSB0aGF0IGxlYWQgdG8gdGhlc2UgZGlzdHJpYnV0aW9ucy4gT25lIGltbWVkaWF0ZSBpbXBsaWNhdGlvbiBvZiBvdXIgYXJndW1lbnQgaXMgdGhhdCBib3RoIHJlc2VhcmNoZXJzIGFuZCBwcmFjdGl0aW9uZXJzIG11c3QgcmVwb3J0IGFuZCBleGFtaW5lIG1vcmUgdGhvcm91Z2hseSB0aGUgZGlzdHJpYnV0aW9uIG9mIHV0aWxpdHkgYmV0d2VlbiBhbmQgd2l0aGluIGRpZmZlcmVudCBzdGFrZWhvbGRlciBncm91cHMuIEhvd2V2ZXIsIGRpc3RyaWJ1dGlvbnMgb2YgdmFyaW91cyBmb3JtcyBhcmlzZSBpbiBtYW55IG1vcmUgYXNwZWN0cyBvZiB0aGUgcmVjb21tZW5kZXIgc3lzdGVtcyBleHBlcmltZW50YWwgcHJvY2VzcywgYW5kIGRpc3RyaWJ1dGlvbmFsIHRoaW5raW5nIGhhcyBzdWJzdGFudGlhbCByYW1pZmljYXRpb25zIGZvciBob3cgd2UgZGVzaWduLCBldmFsdWF0ZSwgYW5kIHByZXNlbnQgcmVjb21tZW5kZXIgc3lzdGVtcyBldmFsdWF0aW9uIGFuZCByZXNlYXJjaCByZXN1bHRzLiBMZXZlcmFnaW5nIGFuZCBlbXBoYXNpemluZyBkaXN0cmlidXRpb25zIGluIHRoZSBldmFsdWF0aW9uIG9mIHJlY29tbWVuZGVyIHN5c3RlbXMgaXMgYSBuZWNlc3Nhcnkgc3RlcCB0byBlbnN1cmUgdGhhdCB0aGUgc3lzdGVtcyBwcm92aWRlIGFwcHJvcHJpYXRlIGFuZCBlcXVpdGFibHktZGlzdHJpYnV0ZWQgYmVuZWZpdCB0byB0aGUgcGVvcGxlIHRoZXkgYWZmZWN0Ln0sCgl1cmxkYXRlID0gezIwMjMtMDktMDd9LAoJam91cm5hbCA9IHtBQ00gVHJhbnNhY3Rpb25zIG9uIFJlY29tbWVuZGVyIFN5c3RlbXN9LAoJYXV0aG9yID0ge0Vrc3RyYW5kLCBNaWNoYWVsIEQuIGFuZCBDYXJ0ZXJldHRlLCBCZW4gYW5kIERpYXosIEZlcm5hbmRvfSwKCW1vbnRoID0gYXVnLAoJeWVhciA9IHsyMDIzfSwKCW5vdGUgPSB7SnVzdCBBY2NlcHRlZH0sCglrZXl3b3JkcyA9IHtkaXN0cmlidXRpb25zLCBldmFsdWF0aW9uLCBleHBvc3VyZSwgc3RhdGlzdGljc30sCn0KCgoKQGFydGljbGV7bWljaGllbHNfZnJhbWV3b3JrXzIwMjMsCgl0aXRsZSA9IHtBIHtGcmFtZXdvcmt9IGFuZCB7VG9vbGtpdH0gZm9yIHtUZXN0aW5nfSB0aGUge0NvcnJlY3RuZXNzfSBvZiB7UmVjb21tZW5kYXRpb259IHtBbGdvcml0aG1zfX0sCgl1cmwgPSB7aHR0cHM6Ly9kb2kub3JnLzEwLjExNDUvMzU5MTEwOX0sCglkb2kgPSB7MTAuMTE0NS8zNTkxMTA5fSwKCWFic3RyYWN0ID0ge0V2YWx1YXRpbmcgcmVjb21tZW5kZXIgc3lzdGVtcyBhZGVxdWF0ZWx5IGFuZCB0aG9yb3VnaGx5IGlzIGFuIGltcG9ydGFudAp0YXNrLiBTaWduaWZpY2FudCBlZmZvcnRzIGFyZSBkZWRpY2F0ZWQgdG8gcHJvcG9zaW5nIG1ldHJpY3MsIG1ldGhvZHMgYW5kCnByb3RvY29scyBmb3IgZG9pbmcgc28uIEhvd2V2ZXIsIHRoZXJlIGhhcyBiZWVuIGxpdHRsZSBkaXNjdXNzaW9uIGluIHRoZQpyZWNvbW1lbmRlciBzeXN0ZW1z4oCZIGxpdGVyYXR1cmUgb24gdGhlIHRvcGljIG9mIHRlc3RpbmcuIEluIHRoaXMgd29yaywgd2UKYWRvcHQgYW5kIGFkYXB0IGNvbmNlcHRzIGZyb20gdGhlIHNvZnR3YXJlIHRlc3RpbmcgZG9tYWluLCBlLmcuLCBjb2RlCmNvdmVyYWdlLCBtZXRhbW9ycGhpYyB0ZXN0aW5nLCBvciBwcm9wZXJ0eS1iYXNlZCB0ZXN0aW5nLCB0byBoZWxwCnJlc2VhcmNoZXJzIHRvIGRldGVjdCBhbmQgY29ycmVjdCBmYXVsdHMgaW4gcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcy4gV2UKcHJvcG9zZSBhIHRlc3Qgc3VpdGUgdGhhdCBjYW4gYmUgdXNlZCB0byB2YWxpZGF0ZSB0aGUgY29ycmVjdG5lc3Mgb2YgYQpyZWNvbW1lbmRhdGlvbiBhbGdvcml0aG0sIGFuZCB0aHVzIGlkZW50aWZ5IGFuZCBjb3JyZWN0IGlzc3VlcyB0aGF0IGNhbgphZmZlY3QgdGhlIHBlcmZvcm1hbmNlIGFuZCBiZWhhdmlvciBvZiB0aGVzZSBhbGdvcml0aG1zLiBPdXIgdGVzdCBzdWl0ZQpjb250YWlucyBib3RoIGJsYWNrIGJveCBhbmQgd2hpdGUgYm94IHRlc3RzIGF0IGV2ZXJ5IGxldmVsIG9mIGFic3RyYWN0aW9uLAppLmUuLCBzeXN0ZW0sIGludGVncmF0aW9uIGFuZCB1bml0LiBUbyBmYWNpbGl0YXRlIGFkb3B0aW9uLCB3ZSByZWxlYXNlClJlY1BhY2sgVGVzdHMsIGFuIG9wZW4tc291cmNlIFB5dGhvbiBwYWNrYWdlIGNvbnRhaW5pbmcgdGVtcGxhdGUgdGVzdAppbXBsZW1lbnRhdGlvbnMuIFdlIHVzZSBpdCB0byB0ZXN0IGZvdXIgcG9wdWxhciBQeXRob24gcGFja2FnZXMgZm9yCnJlY29tbWVuZGVyIHN5c3RlbXM6IFJlY1BhY2ssIFB5TGVuc0tpdCwgU3VycHJpc2UgYW5kIENvcm5hYy4gRGVzcGl0ZSB0aGUKaGlnaCB0ZXN0IGNvdmVyYWdlIG9mIGVhY2ggb2YgdGhlc2UgcGFja2FnZXMsIHdlIGZpbmQgdGhhdCB3ZSBhcmUgc3RpbGwKYWJsZSB0byB1bmNvdmVyIHVuZG9jdW1lbnRlZCBmdW5jdGlvbmFsIHJlcXVpcmVtZW50cyBhbmQgZXZlbiBzb21lIGJ1Z3MuClRoaXMgdmFsaWRhdGVzIG91ciB0aGVzaXMgdGhhdCB0ZXN0aW5nIHRoZSBjb3JyZWN0bmVzcyBvZiByZWNvbW1lbmRhdGlvbgphbGdvcml0aG1zIGNhbiBjb21wbGVtZW50IHRyYWRpdGlvbmFsIG1ldGhvZHMgZm9yIGV2YWx1YXRpbmcKcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcy59LAoJam91cm5hbCA9IHtBQ00gVHJhbnMuIFJlY29tbS4gU3lzdC59LAoJYXV0aG9yID0ge01pY2hpZWxzLCBMaWVuIGFuZCBWZXJhY2h0ZXJ0LCBSb2JpbiBhbmQgRmVycmFybywgQW5kcmVzIGFuZCBGYWxrLCBLaW0gYW5kIEdvZXRoYWxzLCBCYXJ0fSwKCW1vbnRoID0gYXByLAoJeWVhciA9IHsyMDIzfSwKCW5vdGUgPSB7UGxhY2U6IE5ldyBZb3JrLCBOWSwgVVNBClB1Ymxpc2hlcjogQXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAp9CgoKCkBpbnByb2NlZWRpbmdze2Rva291cGlsX2Vhc3lzdHVkeV8yMDIzLAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCglzZXJpZXMgPSB7e1JlY1N5c30gJzIzfSwKCXRpdGxlID0ge3tFYXN5U3R1ZHl9OiB7RnJhbWV3b3JrfSBmb3Ige0Vhc3l9IHtEZXBsb3ltZW50fSBvZiB7VXNlcn0ge1N0dWRpZXN9IG9uIHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN9fSwKCWlzYm4gPSB7OTc5ODQwMDcwMjQxOX0sCglzaG9ydHRpdGxlID0ge3tFYXN5U3R1ZHl9fSwKCXVybCA9IHtodHRwczovL2RsLmFjbS5vcmcvZG9pLzEwLjExNDUvMzYwNDkxNS4zNjEwNjQwfSwKCWRvaSA9IHsxMC4xMTQ1LzM2MDQ5MTUuMzYxMDY0MH0sCglhYnN0cmFjdCA9IHtJbXByb3ZlbWVudHMgaW4gdGhlIHJlY29tbWVuZGVyIHN5c3RlbXMgKFJTKSBkb21haW4gYXJlIG5vdCBwb3NzaWJsZSB3aXRob3V0IGEgdGhvcm91Z2ggd2F5IHRvIGV2YWx1YXRlIGFuZCBjb21wYXJlIG5ld2x5IHByb3Bvc2VkIGFwcHJvYWNoZXMuIFVzZXIgc3R1ZGllcyByZXByZXNlbnQgYSB2aWFibGUgYWx0ZXJuYXRpdmUgdG8gb25saW5lIGFuZCBvZmZsaW5lIGV2YWx1YXRpb24gc2NoZW1lcywgYnV0IGRlc3BpdGUgdGhlaXIgbnVtZXJvdXMgYmVuZWZpdHMsIHRoZXkgYXJlIG9ubHkgcmFyZWx5IHVzZWQuIE9uZSBvZiB0aGUgbWFpbiByZWFzb25zIGJlaGluZCB0aGlzIGZhY3QgaXMgdGhhdCBwcmVwYXJpbmcgYSB1c2VyIHN0dWR5IGZyb20gc2NyYXRjaCBpbnZvbHZlcyBhIGxvdCBvZiBleHRyYSB3b3JrIG9uIHRvcCBvZiBhIHNpbXBsZSBhbGdvcml0aG0gcHJvcG9zYWwuIFRvIHNpbXBsaWZ5IHRoaXMgdGFzaywgd2UgcHJvcG9zZSBFYXN5U3R1ZHksIGEgbW9kdWxhciBmcmFtZXdvcmsgYnVpbHQgb24gdGhlIGNyZWRvIOKAnE1ha2Ugc2ltcGxlIHRoaW5ncyBmYXN0IGFuZCBoYXJkIHRoaW5ncyBwb3NzaWJsZeKAnS4gSXQgZmVhdHVyZXMgcmVhZHktdG8tdXNlIGRhdGFzZXRzLCBwcmVmZXJlbmNlIGVsaWNpdGF0aW9uIG1ldGhvZHMsIGluY3JlbWVudGFsbHkgdHVuZWQgYmFzZWxpbmUgYWxnb3JpdGhtcywgc3R1ZHkgZmxvdyBwbHVnaW5zLCBhbmQgZXZhbHVhdGlvbiBtZXRyaWNzLiBBcyBhIHJlc3VsdCwgYSBzaW1wbGUgc3R1ZHkgY29tcGFyaW5nIHNldmVyYWwgUlMgY2FuIGJlIGRlcGxveWVkIHdpdGgganVzdCBhIGZldyBjbGlja3MsIHdoaWxlIG1vcmUgY29tcGxleCBzdHVkeSBkZXNpZ25zIGNhbiBzdGlsbCBiZW5lZml0IGZyb20gYSByYW5nZSBvZiByZXVzYWJsZSBjb21wb25lbnRzLCBzdWNoIGFzIHByZWZlcmVuY2UgZWxpY2l0YXRpb24uIE92ZXJhbGwsIEVhc3lTdHVkeSBkcmFtYXRpY2FsbHkgZGVjcmVhc2VzIHRoZSBnYXAgYmV0d2VlbiB0aGUgbGFib3Jpb3VzbmVzcyBvZiBvZmZsaW5lIGV2YWx1YXRpb24gdnMuIHVzZXIgc3R1ZGllcyBhbmQsIHRoZXJlZm9yZSwgbWF5IGNvbnRyaWJ1dGUgdG93YXJkcyB0aGUgbW9yZSByZWxpYWJsZSBhbmQgaW5zaWdodGZ1bCB1c2VyLWNlbnRyaWMgZXZhbHVhdGlvbiBvZiBuZXh0LWdlbmVyYXRpb24gUlMuIFRoZSBwcm9qZWN0IHJlcG9zaXRvcnkgaXMgYXZhaWxhYmxlIGZyb20gaHR0cHM6Ly9iaXQubHkvZWFzeS1zdHVkeS1yZXBvLn0sCgl1cmxkYXRlID0gezIwMjMtMDktMTh9LAoJYm9va3RpdGxlID0ge1Byb2NlZWRpbmdzIG9mIHRoZSAxN3RoIHtBQ019IHtDb25mZXJlbmNlfSBvbiB7UmVjb21tZW5kZXJ9IHtTeXN0ZW1zfX0sCglwdWJsaXNoZXIgPSB7QXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAoJYXV0aG9yID0ge0Rva291cGlsLCBQYXRyaWsgYW5kIFBlc2thLCBMYWRpc2xhdn0sCgltb250aCA9IHNlcCwKCXllYXIgPSB7MjAyM30sCglrZXl3b3JkcyA9IHtSZWNvbW1lbmRlciBzeXN0ZW1zLCBldmFsdWF0aW9uIGZyYW1ld29ya3MsIHVzZXIgY2VudHJpYywgdXNlciBzdHVkaWVzfSwKCXBhZ2VzID0gezExOTYtLTExOTl9LAp9CgoKCkBhcnRpY2xle3BhdGhha191bmRlcnN0YW5kaW5nXzIwMjMsCgl0aXRsZSA9IHtVbmRlcnN0YW5kaW5nIHRoZSB7Q29udHJpYnV0aW9ufSBvZiB7UmVjb21tZW5kYXRpb259IHtBbGdvcml0aG1zfSBvbiB7TWlzaW5mb3JtYXRpb259IHtSZWNvbW1lbmRhdGlvbn0gYW5kIHtNaXNpbmZvcm1hdGlvbn0ge0Rpc3NlbWluYXRpb259IG9uIHtTb2NpYWx9IHtOZXR3b3Jrc319LAoJaXNzbiA9IHsxNTU5LTExMzF9LAoJdXJsID0ge2h0dHBzOi8vZGwuYWNtLm9yZy9kb2kvMTAuMTE0NS8zNjE2MDg4fSwKCWRvaSA9IHsxMC4xMTQ1LzM2MTYwODh9LAoJYWJzdHJhY3QgPSB7U29jaWFsIG5ldHdvcmtzIGFyZSBhIHBsYXRmb3JtIGZvciBpbmRpdmlkdWFscyBhbmQgb3JnYW5pemF0aW9ucyB0byBjb25uZWN0IHdpdGggZWFjaCBvdGhlciBhbmQgaW5mb3JtLCBhZHZlcnRpc2UsIHNwcmVhZCBpZGVhcywgYW5kIHVsdGltYXRlbHkgaW5mbHVlbmNlIG9waW5pb25zLiBUaGVzZSBwbGF0Zm9ybXMgaGF2ZSBiZWVuIGtub3duIHRvIHByb3BlbCBtaXNpbmZvcm1hdGlvbi4gV2UgYXJndWUgdGhhdCB0aGlzIGNvdWxkIGJlIGNvbXBvdW5kZWQgYnkgdGhlIHJlY29tbWVuZGVyIGFsZ29yaXRobXMgdGhhdCB0aGVzZSBwbGF0Zm9ybXMgdXNlIHRvIHN1Z2dlc3QgaXRlbXMgcG90ZW50aWFsbHkgb2YgaW50ZXJlc3QgdG8gdGhlaXIgdXNlcnMsIGdpdmVuIHRoZSBrbm93biBiaWFzZXMgYW5kIGZpbHRlciBidWJibGVzIGlzc3VlcyBhZmZlY3RpbmcgcmVjb21tZW5kZXIgc3lzdGVtcy4gV2hpbGUgbXVjaCBoYXMgYmVlbiBzdHVkaWVkIGFib3V0IG1pc2luZm9ybWF0aW9uIG9uIHNvY2lhbCBuZXR3b3JrcywgdGhlIHBvdGVudGlhbCBleGFjZXJiYXRpb24gdGhhdCBjb3VsZCByZXN1bHQgZnJvbSByZWNvbW1lbmRlciBhbGdvcml0aG1zIGluIHRoaXMgZW52aXJvbm1lbnQgaXMgaW4gaXRzIGluZmFuY3kuIEluIHRoaXMgbWFudXNjcmlwdCwgd2UgcHJlc2VudCB0aGUgcmVzdWx0IG9mIGFuIGluLWRlcHRoIGFuYWx5c2lzIGNvbmR1Y3RlZCBvbiB0d28gZGF0YXNldHMgKFBvbGl0aWZhY3QgRmFrZU5ld3NOZXQgZGF0YXNldCBhbmQgSGVhbHRoU3RvcnkgRmFrZUhlYWx0aCBkYXRhc2V0KSBpbiBvcmRlciB0byBkZWVwZW4gb3VyIHVuZGVyc3RhbmRpbmcgb2YgdGhlIGludGVyY29ubmVjdGlvbiBiZXR3ZWVuIHJlY29tbWVuZGVyIGFsZ29yaXRobXMgYW5kIG1pc2luZm9ybWF0aW9uIHNwcmVhZCBvbiBUd2l0dGVyLiBJbiBwYXJ0aWN1bGFyLCB3ZSBleHBsb3JlIHRoZSBkZWdyZWUgdG8gd2hpY2ggd2VsbC1rbm93biByZWNvbW1lbmRhdGlvbiBhbGdvcml0aG1zIGFyZSBwcm9uZSB0byBiZSBpbXBhY3RlZCBieSBtaXNpbmZvcm1hdGlvbi4gVmlhIHNpbXVsYXRpb24sIHdlIGFsc28gc3R1ZHkgbWlzaW5mb3JtYXRpb24gZGlmZnVzaW9uIG9uIHNvY2lhbCBuZXR3b3JrcywgYXMgdHJpZ2dlcmVkIGJ5IHN1Z2dlc3Rpb25zIHByb2R1Y2VkIGJ5IHRoZXNlIHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobXMuIE91dGNvbWVzIGZyb20gdGhpcyB3b3JrIGV2aWRlbmNlIHRoYXQgbWlzaW5mb3JtYXRpb24gZG9lcyBub3QgZXF1YWxseSBhZmZlY3QgYWxsIHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobXMuIFBvcHVsYXJpdHktYmFzZWQgYW5kIG5ldHdvcmstYmFzZWQgcmVjb21tZW5kZXIgYWxnb3JpdGhtcyBjb250cmlidXRlIHRoZSBtb3N0IHRvIG1pc2luZm9ybWF0aW9uIGRpZmZ1c2lvbi4gVXNlcnMgd2hvIGFyZSBrbm93biB0byBiZSBzdXBlcnNwcmVhZGVycyBhcmUga25vd24gdG8gZGlyZWN0bHkgaW1wYWN0IGFsZ29yaXRobWljIHBlcmZvcm1hbmNlIGFuZCBtaXNpbmZvcm1hdGlvbiBzcHJlYWQgaW4gc3BlY2lmaWMgc2NlbmFyaW9zLiBGaW5kaW5ncyBlbWVyZ2luZyBmcm9tIG91ciBleHBsb3JhdGlvbiByZXN1bHQgaW4gYSBudW1iZXIgb2YgaW1wbGljYXRpb25zIGZvciByZXNlYXJjaGVycyBhbmQgcHJhY3RpdGlvbmVycyB0byBjb25zaWRlciB3aGVuIGRlc2lnbmluZyBhbmQgZGVwbG95aW5nIHJlY29tbWVuZGVyIGFsZ29yaXRobXMgaW4gc29jaWFsIG5ldHdvcmtzLn0sCgl1cmxkYXRlID0gezIwMjMtMDktMTl9LAoJam91cm5hbCA9IHtBQ00gVHJhbnNhY3Rpb25zIG9uIHRoZSBXZWJ9LAoJYXV0aG9yID0ge1BhdGhhaywgUm95YWwgYW5kIFNwZXp6YW5vLCBGcmFuY2VzY2EgYW5kIFBlcmEsIE1hcmlhIFNvbGVkYWR9LAoJbW9udGggPSBhdWcsCgl5ZWFyID0gezIwMjN9LAoJbm90ZSA9IHtKdXN0IEFjY2VwdGVkfSwKCWtleXdvcmRzID0ge1R3aXR0ZXIsIGRpZmZ1c2lvbiwgbWlzaW5mb3JtYXRpb24sIG5ld3MsIHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobXMsIHNvY2lhbCBuZXR3b3Jrc30sCn0KCgoKQGFydGljbGV7Z29kaW5vdF9tZWFzdXJpbmdfMjAyMywKCXRpdGxlID0ge01lYXN1cmluZyB0aGUgZWZmZWN0IG9mIGNvbGxhYm9yYXRpdmUgZmlsdGVyaW5nIG9uIHRoZSBkaXZlcnNpdHkgb2YgdXNlcnPigJkgYXR0ZW50aW9ufSwKCXZvbHVtZSA9IHs4fSwKCWNvcHlyaWdodCA9IHsyMDIzIFRoZSBBdXRob3Iocyl9LAoJaXNzbiA9IHsyMzY0LTgyMjh9LAoJdXJsID0ge2h0dHBzOi8vYXBwbGllZG5ldHNjaS5zcHJpbmdlcm9wZW4uY29tL2FydGljbGVzLzEwLjEwMDcvczQxMTA5LTAyMi0wMDUzMC03fSwKCWRvaSA9IHsxMC4xMDA3L3M0MTEwOS0wMjItMDA1MzAtN30sCglhYnN0cmFjdCA9IHtXaGlsZSB0aGUgZXZlci1pbmNyZWFzaW5nIGVtZXJnZW5jZSBvZiBvbmxpbmUgc2VydmljZXMgaGFzIGxlZCB0byBhIGdyb3dpbmcgaW50ZXJlc3QgaW4gdGhlIGRldmVsb3BtZW50IG9mIHJlY29tbWVuZGVyIHN5c3RlbXMsIHRoZSBhbGdvcml0aG1zIHVuZGVycGlubmluZyBzdWNoIHN5c3RlbXMgaGF2ZSBiZWd1biB0byBiZSBjcml0aWNpemVkIGZvciB0aGVpciByb2xlIGluIGxpbWl0aW5nIHRoZSB2YXJpZXR5IG9mIGNvbnRlbnQgZXhwb3NlZCB0byB1c2Vycy4gSW4gdGhpcyBjb250ZXh0LCB0aGUgbm90aW9uIG9mIGRpdmVyc2l0eSBoYXMgYmVlbiBwcm9wb3NlZCBhcyBhIHdheSBvZiBtaXRpZ2F0aW5nIHRoZSBzaWRlIGVmZmVjdHMgcmVzdWx0aW5nIGZyb20gdGhlIHNwZWNpYWxpemF0aW9uIG9mIHJlY29tbWVuZGVyIHN5c3RlbXMuIEluIHRoaXMgcGFwZXIsIHVzaW5nIGEgd2VsbC1rbm93biByZWNvbW1lbmRlciBzeXN0ZW0gdGhhdCBtYWtlcyB1c2Ugb2YgY29sbGFib3JhdGl2ZSBmaWx0ZXJpbmcgaW4gdGhlIGNvbnRleHQgb2YgbXVzaWNhbCBjb250ZW50LCB3ZSBhbmFseXplIHRoZSBkaXZlcnNpdHkgb2YgcmVjb21tZW5kYXRpb25zIGdlbmVyYXRlZCB0aHJvdWdoIHRoZSBsZW5zIG9mIHRoZSByZWNlbnRseSBwcm9wb3NlZCBpbmZvcm1hdGlvbiBuZXR3b3JrIGRpdmVyc2l0eSBtZWFzdXJlLiBUaGUgcmVzdWx0cyBvZiBvdXIgc3R1ZHkgb2ZmZXIgc2lnbmlmaWNhbnQgaW5zaWdodHMgaW50byB0aGUgZWZmZWN0IG9mIGFsZ29yaXRobWljIHJlY29tbWVuZGF0aW9ucy4gT24gdGhlIG9uZSBoYW5kLCB3ZSBzaG93IHRoYXQgdGhlIG11c2ljYWwgc2VsZWN0aW9ucyBvZiBhIGxhcmdlIHByb3BvcnRpb24gb2YgdXNlcnMgYXJlIGRpdmVyc2lmaWVkIGFzIGEgcmVzdWx0IG9mIHRoZSByZWNvbW1lbmRhdGlvbnMuIE9uIHRoZSBvdGhlciBoYW5kLCBob3dldmVyLCBzdWNoIGltcHJvdmVtZW50cyBkbyBub3QgYmVuZWZpdCBhbGwgdXNlcnMuIFRoZXkgYXJlIGluIGZhY3QgbWFpbmx5IHJlc3RyaWN0ZWQgdG8gdXNlcnMgd2l0aCBhIGxvdyBsZXZlbCBvZiBhY3Rpdml0eSBvciB3aG9zZSBwYXN0IG11c2ljYWwgbGlzdGVuaW5nIHNlbGVjdGlvbnMgYXJlIHZlcnkgbmFycm93LiBUaHJvdWdoIG1vcmUgaW4tZGVwdGggaW52ZXN0aWdhdGlvbnMsIHdlIGFsc28gZGlzY292ZXJlZCB0aGF0IHdoaWxlIHJlY29tbWVuZGF0aW9ucyBnZW5lcmFsbHkgaW5jcmVhc2UgdGhlIHZhcmlldHkgb2YgdGhlIHNvbmdzIHJlY29tbWVuZGVkIHRvIHVzZXJzLCB0aGV5IG5vbmV0aGVsZXNzIGZhaWwgdG8gcHJvdmlkZSBhIGJhbGFuY2VkIGV4cG9zdXJlIHRvIHRoZSBkaWZmZXJlbnQgcmVsYXRlZCBjYXRlZ29yaWVzLn0sCglsYW5ndWFnZSA9IHtlbn0sCgludW1iZXIgPSB7MX0sCgl1cmxkYXRlID0gezIwMjMtMDktMTl9LAoJam91cm5hbCA9IHtBcHBsaWVkIE5ldHdvcmsgU2NpZW5jZX0sCglhdXRob3IgPSB7R29kaW5vdCwgQXVndXN0aW4gYW5kIFRhcmlzc2FuLCBGYWJpZW59LAoJbW9udGggPSBkZWMsCgl5ZWFyID0gezIwMjN9LAoJbm90ZSA9IHtOdW1iZXI6IDEKUHVibGlzaGVyOiBTcHJpbmdlck9wZW59LAoJcGFnZXMgPSB7MS0tMTh9LAp9CgoKCkBpbnByb2NlZWRpbmdze3RpYW5fZXN0aW1hdGluZ18yMDIwLAoJc2VyaWVzID0ge3tDSElJUn0gJzIwfSwKCXRpdGxlID0ge0VzdGltYXRpbmcge0Vycm9yfSBhbmQge0JpYXN9IGluIHtPZmZsaW5lfSB7RXZhbHVhdGlvbn0ge1Jlc3VsdHN9fSwKCXVybCA9IHtodHRwczovL2RvaS5vcmcvMTAuMTE0NS8zMzQzNDEzLjMzNzgwMDR9LAoJZG9pID0gezEwLjExNDUvMzM0MzQxMy4zMzc4MDA0fSwKCXVybGRhdGUgPSB7MjAyMC0wMy0yNn0sCglib29rdGl0bGUgPSB7UHJvY2VlZGluZ3Mgb2YgdGhlIDIwMjAge0NvbmZlcmVuY2V9IG9uIHtIdW1hbn0ge0luZm9ybWF0aW9ufSB7SW50ZXJhY3Rpb259IGFuZCB7UmV0cmlldmFsfX0sCglwdWJsaXNoZXIgPSB7QUNNfSwKCWF1dGhvciA9IHtUaWFuLCBNdWN1biBhbmQgRWtzdHJhbmQsIE1pY2hhZWwgRH0sCgltb250aCA9IG1hciwKCXllYXIgPSB7MjAyMH0sCglub3RlID0ge0pvdXJuYWwgQWJicmV2aWF0aW9uOiBDSElJUiAnMjB9LAoJa2V5d29yZHMgPSB7b2ZmbGluZSBldmFsdWF0aW9uLCBzaW11bGF0aW9ufSwKCXBhZ2VzID0gezM5Mi0tMzk2fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tkb2tvdXBpbF9lZmZlY3RfMjAyMywKCWFkZHJlc3MgPSB7TmV3IFlvcmssIE5ZLCBVU0F9LAoJc2VyaWVzID0ge3tVTUFQfSAnMjMge0FkanVuY3R9fSwKCXRpdGxlID0ge1RoZSB7RWZmZWN0fSBvZiB7U2ltaWxhcml0eX0ge01ldHJpY30gYW5kIHtHcm91cH0ge1NpemV9IG9uIHtPdXRsaWVyfSB7U2VsZWN0aW9ufSBcJiB7U2F0aXNmYWN0aW9ufSBpbiB7R3JvdXB9IHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN9fSwKCWlzYm4gPSB7OTc4LTEtNDUwMy05ODkxLTZ9LAoJdXJsID0ge2h0dHBzOi8vZGwuYWNtLm9yZy9kb2kvMTAuMTE0NS8zNTYzMzU5LjM1OTczODZ9LAoJZG9pID0gezEwLjExNDUvMzU2MzM1OS4zNTk3Mzg2fSwKCWFic3RyYWN0ID0ge0dyb3VwIHJlY29tbWVuZGVyIHN5c3RlbXMgKEdSUykgYXJlIGEgc3BlY2lmaWMgY2FzZSBvZiByZWNvbW1lbmRlciBzeXN0ZW1zIChSUyksIHdoZXJlIHJlY29tbWVuZGF0aW9ucyBhcmUgY29uc3RydWN0ZWQgdG8gYSBncm91cCBvZiB1c2VycyByYXRoZXIgdGhhbiBhbiBpbmRpdmlkdWFsLiBHUlMgaGFzIGRpdmVyc2UgYXBwbGljYXRpb24gYXJlYXMgaW5jbHVkaW5nIHRyaXAgcGxhbm5pbmcsIHJlY29tbWVuZGluZyBtb3ZpZXMgdG8gd2F0Y2ggdG9nZXRoZXIsIG9yIG11c2ljIGluIHNoYXJlZCBlbnZpcm9ubWVudHMuIEhvd2V2ZXIsIGR1ZSB0byB0aGUgbGFjayBvZiBsYXJnZSBkYXRhc2V0cyB3aXRoIGdyb3VwIGRlY2lzaW9uLW1ha2luZyBmZWVkYmFjayBpbmZvcm1hdGlvbiwgb3IgZXZlbiB0aGUgZ3JvdXAgZGVmaW5pdGlvbnMsIEdSUyBhcHByb2FjaGVzIGFyZSBvZnRlbiBldmFsdWF0ZWQgb2ZmbGluZSB3LnIudC4gaW5kaXZpZHVhbCB1c2VyIGZlZWRiYWNrIGFuZCBhcnRpZmljaWFsbHkgZ2VuZXJhdGVkIGdyb3Vwcy4gVGhlc2Ugc3ludGhldGljIGdyb3VwcyBhcmUgdXN1YWxseSBjb25zdHJ1Y3RlZCB3LnIudC4gcHJlLWRlZmluZWQgZ3JvdXAgc2l6ZSBhbmQgaW50ZXItdXNlciBzaW1pbGFyaXR5IG1ldHJpYy4gV2hpbGUgbnVtZXJvdXMgdmFyaWFudHMgb2Ygc3ludGhldGljIGdyb3VwIGdlbmVyYXRpb24gcHJvY2VkdXJlcyB3ZXJlIHV0aWxpemVkIHNvIGZhciwgaXRzIGltcGFjdCBvbiB0aGUgZXZhbHVhdGlvbiByZXN1bHRzIHdhcyBub3Qgc3VmZmljaWVudGx5IGRpc2N1c3NlZC4gSW4gdGhpcyBwYXBlciwgd2UgYWRkcmVzcyB0aGlzIHJlc2VhcmNoIGdhcCBieSBpbnZlc3RpZ2F0aW5nIHRoZSBpbXBhY3Qgb2YgdmFyaW91cyBzeW50aGV0aWMgZ3JvdXAgZ2VuZXJhdGlvbiBwcm9jZWR1cmVzLCBuYW1lbHkgdGhlIHVzYWdlIG9mIGRpZmZlcmVudCB1c2VyIHNpbWlsYXJpdHkgbWV0cmljcyBhbmQgdGhlIGVmZmVjdCBvZiBncm91cCBzaXplcy4gV2UgY29uc2lkZXIgdGhlbSBpbiB0aGUgY29udGV4dCBvZiDigJxvdXRsaWVyIHZzLiBtYWpvcml0eeKAnSBncm91cHMsIHdoZXJlIGEgZ3JvdXAgb2Ygc2ltaWxhciB1c2VycyBpcyBleHRlbmRlZCB3aXRoIG9uZSBvciBtb3JlIGRpdmVyc2Ugb25lcy4gRXhwZXJpbWVudGFsIHJlc3VsdHMgaW5kaWNhdGUgYSBzdHJvbmcgaW1wYWN0IG9mIHRoZSBzZWxlY3RlZCBzaW1pbGFyaXR5IG1ldHJpYyBvbiBib3RoIHRoZSB0eXBpY2FsIGNoYXJhY3RlcmlzdGljcyBvZiBzZWxlY3RlZCBvdXRsaWVycyBhcyB3ZWxsIGFzIHRoZSBwZXJmb3JtYW5jZSBvZiBpbmRpdmlkdWFsIEdSUyBhbGdvcml0aG1zLiBNb3Jlb3Zlciwgd2Ugc2hvdyB0aGF0IGNlcnRhaW4gYWxnb3JpdGhtcyBiZXR0ZXIgYWRhcHQgdG8gbGFyZ2VyIGdyb3VwcyB0aGFuIG90aGVycy59LAoJdXJsZGF0ZSA9IHsyMDIzLTA2LTI1fSwKCWJvb2t0aXRsZSA9IHtBZGp1bmN0IHtQcm9jZWVkaW5nc30gb2YgdGhlIDMxc3Qge0FDTX0ge0NvbmZlcmVuY2V9IG9uIHtVc2VyfSB7TW9kZWxpbmd9LCB7QWRhcHRhdGlvbn0gYW5kIHtQZXJzb25hbGl6YXRpb259fSwKCXB1Ymxpc2hlciA9IHtBc3NvY2lhdGlvbiBmb3IgQ29tcHV0aW5nIE1hY2hpbmVyeX0sCglhdXRob3IgPSB7RG9rb3VwaWwsIFBhdHJpayBhbmQgUGVza2EsIExhZGlzbGF2fSwKCW1vbnRoID0ganVuLAoJeWVhciA9IHsyMDIzfSwKCWtleXdvcmRzID0ge0dyb3VwIFJlY29tbWVuZGVyIHN5c3RlbXMsIFN5bnRoZXRpYyBncm91cHMgY29uc3RydWN0aW9uLCBVc2VyIHNpbWlsYXJpdHkgbWV0cmljc30sCglwYWdlcyA9IHsyOTYtLTMwMX0sCn0KCgoKQGlucHJvY2VlZGluZ3N7YmF1ZXJfZmFpcnJlY2tpdF8yMDIzLAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCgl0aXRsZSA9IHt7RmFpclJlY0tpdH06IHtBfSB7V2VifS1iYXNlZCB7QW5hbHlzaXN9IHtTb2Z0d2FyZX0gZm9yIHtSZWNvbW1lbmRlcn0ge0V2YWx1YXRpb25zfX0sCgl1cmwgPSB7aHR0cHM6Ly9kb2kub3JnLzEwLjExNDUvMzU3Njg0MC4zNTc4Mjc0fSwKCWRvaSA9IHsxMC4xMTQ1LzM1NzY4NDAuMzU3ODI3NH0sCglhYnN0cmFjdCA9IHtGYWlyUmVjS2l0IGlzIGEgd2ViLWJhc2VkIGFuYWx5c2lzIHNvZnR3YXJlIHRoYXQgc3VwcG9ydHMgcmVzZWFyY2hlcnMgaW4KcGVyZm9ybWluZywgYW5hbHl6aW5nLCBhbmQgdW5kZXJzdGFuZGluZyByZWNvbW1lbmRhdGlvbiBjb21wdXRhdGlvbnMuIFRoZQppZGVhIGJlaGluZCBGYWlyUmVjS2l0IGlzIHRvIGZhY2lsaXRhdGUgdGhlIGluLWRlcHRoIGFuYWx5c2lzIG9mCnJlY29tbWVuZGF0aW9uIG91dGNvbWVzIGNvbnNpZGVyaW5nIGZhaXJuZXNzIGFzcGVjdHMuIFdpdGggKG5lc3RlZCkKZmlsdGVycyBvbiB1c2VyIG9yIGl0ZW0gYXR0cmlidXRlcywgbWV0cmljcyBjYW4gZWFzaWx5IGJlIGNvbXBhcmVkIGFjcm9zcwp1c2VyIGFuZCBpdGVtIHN1Ymdyb3Vwcy4gRnVydGhlciwgKG5lc3RlZCkgZmlsdGVycyBjYW4gYmUgdXNlZCBvbiB0aGUKZGF0YXNldCBsZXZlbDsgdGhpcyB3YXksIHJlY29tbWVuZGF0aW9uIG91dGNvbWVzIGNhbiBiZSBjb21wYXJlZCBhY3Jvc3MKc2V2ZXJhbCBzdWItZGF0YXNldHMgdG8gYW5hbHl6ZSBmb3IgZGlmZmVyZW5jZXMgY29uc2lkZXJpbmcgZmFpcm5lc3MKYXNwZWN0cy4gVGhlIHNvZnR3YXJlIGN1cnJlbnRseSBmZWF0dXJlcyBmaXZlIGRhdGFzZXRzLCAxMSBtZXRyaWNzLCBhbmQgMjEKcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcyB0byBiZSB1c2VkIGluIGNvbXB1dGF0aW9uYWwgZXhwZXJpbWVudGF0aW9uLiBJdAppcyBvcGVuIHNvdXJjZSBhbmQgZGV2ZWxvcGVkIGluIGEgbW9kdWxhciBtYW5uZXIgdG8gZmFjaWxpdGF0ZSBleHRlbnNpb24uClRoZSBhbmFseXNpcyBzb2Z0d2FyZSBjb25zaXN0cyBvZiB0d28gY29tcG9uZW50czogQSBzb2Z0d2FyZSBwYWNrYWdlCihGYWlyUmVjS2l0TGliKSBmb3IgcnVubmluZyByZWNvbW1lbmRhdGlvbiBhbGdvcml0aG1zIG9uIHRoZSBhdmFpbGFibGUKZGF0YXNldHMgYW5kIGEgd2ViLWJhc2VkIHVzZXIgaW50ZXJmYWNlIChGYWlyUmVjS2l0QXBwKSB0byBzdGFydApleHBlcmltZW50cywgcmV0cmlldmUgcmVzdWx0cyBvZiBwcmV2aW91cyBleHBlcmltZW50cywgYW5kIGFuYWx5emUKZGV0YWlscy4gVGhlIGFwcGxpY2F0aW9uIGFsc28gY29tZXMgd2l0aCBleHRlbnNpdmUgZG9jdW1lbnRhdGlvbiBhbmQKb3B0aW9ucyBmb3IgcmVzdWx0IGN1c3RvbWl6YXRpb24sIHdoaWNoIG1ha2VzIGZvciBhIGZsZXhpYmxlIHRvb2wgdGhhdApzdXBwb3J0cyBpbi1kZXB0aCBhbmFseXNpcy59LAoJdXJsZGF0ZSA9IHsyMDIzLTAzLTI4fSwKCWJvb2t0aXRsZSA9IHt7Q0hJSVJ9ICcyM30sCglwdWJsaXNoZXIgPSB7QXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAoJYXV0aG9yID0ge0JhdWVyLCBDaHJpc3RpbmUgYW5kIENodW5nLCBMZW5uYXJkIGFuZCBDb3JuZWxpc3NlbiwgQWxla3NlaiBhbmQgdmFuIERyaWVzc2VsLCBJc2FiZWxsZSBhbmQgdmFuIGRlciBIb29ybiwgRGllZGUgYW5kIGRlIEpvbmcsIFltZSBhbmQgTGUsIExhbiBhbmQgTmFqaXlhbiBUYWJyaXosIFNhbmF6IGFuZCBTcGFhbnMsIFJvZGVyaWNrIGFuZCBUaGlqc2VuLCBDYXNwZXIgYW5kIFZlcmJlZXRlbiwgUm9iZXJ0IGFuZCBXZXNzZWxpbmcsIFZvcyBhbmQgV2llbGFuZCwgRmVybn0sCgltb250aCA9IG1hciwKCXllYXIgPSB7MjAyM30sCglub3RlID0ge0pvdXJuYWwgQWJicmV2aWF0aW9uOiBDSElJUiAnMjN9LAoJa2V5d29yZHMgPSB7ZXZhbHVhdGlvbiwgbXVzaWMsIHdlYi1iYXNlZCwgcmVjb21tZW5kZXIgc3lzdGVtcywgdG9vbGtpdCwgYW5hbHlzaXMsIG1vdmllcywgcmVzb3VyY2V9LAoJcGFnZXMgPSB7NDM4LS00NDN9LAp9CgoKCkBhcnRpY2xle2JhbGNoYW5vd3NraV9jb21wYXJhdGl2ZV8yMDIzLAoJdGl0bGUgPSB7QSB7Q29tcGFyYXRpdmV9IHtTdHVkeX0gb2Yge1Jhbmt9IHtBZ2dyZWdhdGlvbn0ge01ldGhvZHN9IGluIHtSZWNvbW1lbmRhdGlvbn0ge1N5c3RlbXN9fSwKCXZvbHVtZSA9IHsyNX0sCglpc3NuID0gezEwOTktNDMwMH0sCgl1cmwgPSB7aHR0cHM6Ly93d3cubWRwaS5jb20vMTA5OS00MzAwLzI1LzEvMTMyfSwKCWRvaSA9IHsxMC4zMzkwL2UyNTAxMDEzMn0sCglhYnN0cmFjdCA9IHtUaGUgYWltIG9mIGEgcmVjb21tZW5kZXIgc3lzdGVtIGlzIHRvIHN1Z2dlc3QgdG8gdGhlIHVzZXIgY2VydGFpbiBwcm9kdWN0cwpvciBzZXJ2aWNlcyB0aGF0IG1vc3QgbGlrZWx5IHdpbGwgaW50ZXJlc3QgdGhlbS4gV2l0aGluIHRoZSBjb250ZXh0IG9mCnBlcnNvbmFsaXplZCByZWNvbW1lbmRlciBzeXN0ZW1zLCBhIG51bWJlciBvZiBhbGdvcml0aG1zIGhhdmUgYmVlbgpzdWdnZXN0ZWQgdG8gZ2VuZXJhdGUgYSByYW5raW5nIG9mIGl0ZW1zIHRhaWxvcmVkIHRvIGluZGl2aWR1YWwgdXNlcgpwcmVmZXJlbmNlcy4gSG93ZXZlciwgdGhlc2UgYWxnb3JpdGhtcyBkbyBub3QgZ2VuZXJhdGUgaWRlbnRpY2FsCnJlY29tbWVuZGF0aW9ucywgYW5kIGZvciB0aGlzIHJlYXNvbiBpdCBoYXMgYmVlbiBzdWdnZXN0ZWQgaW4gdGhlCmxpdGVyYXR1cmUgdGhhdCB0aGUgcmVzdWx0cyBvZiB0aGVzZSBhbGdvcml0aG1zIGNhbiBiZSBjb21iaW5lZCB1c2luZwphZ2dyZWdhdGlvbiB0ZWNobmlxdWVzLCBob3BpbmcgdGhhdCB0aGlzIHdpbGwgdHJhbnNsYXRlIGludG8gYW4KaW1wcm92ZW1lbnQgaW4gdGhlIHF1YWxpdHkgb2YgdGhlIGZpbmFsIHJlY29tbWVuZGF0aW9uLiBJbiBvcmRlciB0byBzZWUKd2hpY2ggb2YgdGhlc2UgdGVjaG5pcXVlcyBpbmNyZWFzZSB0aGUgcXVhbGl0eSBvZiByZWNvbW1lbmRhdGlvbnMgdG8gdGhlCmdyZWF0ZXN0IGV4dGVudCwgdGhlIGF1dGhvcnMgb2YgdGhpcyBwdWJsaWNhdGlvbiBjb25kdWN0ZWQgZXhwZXJpbWVudHMgaW4Kd2hpY2ggdGhleSBjb25zaWRlcmVkIGZpdmUgcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcyBhbmQgMjAgYWdncmVnYXRpb24KbWV0aG9kcy4gVGhlIHJlc2VhcmNoIHdhcyBjYXJyaWVkIG91dCBvbiB0aGUgcG9wdWxhciBhbmQgcHVibGljbHkKYXZhaWxhYmxlIE1vdmllTGVucyAxMDBrIGFuZCBNb3ZpZUxlbnMgMU0gZGF0YXNldHMsIGFuZCB0aGUgcmVzdWx0cyB3ZXJlCmNvbmZpcm1lZCBieSBzdGF0aXN0aWNhbCB0ZXN0cy59LAoJbnVtYmVyID0gezF9LAoJdXJsZGF0ZSA9IHsyMDIzLTAxLTEyfSwKCWpvdXJuYWwgPSB7RW50cm9weX0sCglhdXRob3IgPSB7QmHFgmNoYW5vd3NraSwgTWljaGHFgiBhbmQgQm9yeWN6a2EsIFVyc3p1bGF9LAoJbW9udGggPSBqYW4sCgl5ZWFyID0gezIwMjN9LAoJbm90ZSA9IHtQdWJsaXNoZXI6IE11bHRpZGlzY2lwbGluYXJ5IERpZ2l0YWwgUHVibGlzaGluZyBJbnN0aXR1dGV9LAoJcGFnZXMgPSB7MTMyfSwKfQoKCgpAdW5wdWJsaXNoZWR7YmhhdHRhY2hhcnlhX3doYXRfMjAyMiwKCXRpdGxlID0ge1doYXQge1lvdX0ge0xpa2V9OiB7R2VuZXJhdGluZ30ge0V4cGxhaW5hYmxlfSB7VG9waWNhbH0ge1JlY29tbWVuZGF0aW9uc30gZm9yIHtUd2l0dGVyfSB7VXNpbmd9IHtTb2NpYWx9IHtBbm5vdGF0aW9uc319LAoJdXJsID0ge2h0dHA6Ly9hcnhpdi5vcmcvYWJzLzIyMTIuMTM4OTd9LAoJYWJzdHJhY3QgPSB7V2l0aCBvdmVyIDUwMCBtaWxsaW9uIHR3ZWV0cyBwb3N0ZWQgcGVyIGRheSwgaW4gVHdpdHRlciwgaXQgaXMgZGlmZmljdWx0CmZvciBUd2l0dGVyIHVzZXJzIHRvIGRpc2NvdmVyIGludGVyZXN0aW5nIGNvbnRlbnQgZnJvbSB0aGUgZGVsdWdlIG9mCnVuaW50ZXJlc3RpbmcgcG9zdHMuIEluIHRoaXMgd29yaywgd2UgcHJlc2VudCBhIG5vdmVsLCBleHBsYWluYWJsZSwKdG9waWNhbCByZWNvbW1lbmRhdGlvbiBzeXN0ZW0sIHRoYXQgdXRpbGl6ZXMgc29jaWFsIGFubm90YXRpb25zLCB0byBoZWxwClR3aXR0ZXIgdXNlcnMgZGlzY292ZXIgdHdlZXRzLCBvbiB0b3BpY3Mgb2YgdGhlaXIgaW50ZXJlc3QuIEEgbWFqb3IKY2hhbGxlbmdlIGluIHVzaW5nIHRyYWRpdGlvbmFsIHJhdGluZyBkZXBlbmRlbnQgcmVjb21tZW5kYXRpb24gc3lzdGVtcywKbGlrZSBjb2xsYWJvcmF0aXZlIGZpbHRlcmluZyBhbmQgY29udGVudCBiYXNlZCBzeXN0ZW1zLCBpbiBoaWdoIHZvbHVtZQpzb2NpYWwgbmV0d29ya3MgaXMgdGhhdCwgZHVlIHRvIGF0dGVudGlvbiBzY2FyY2l0eSBtb3N0IGl0ZW1zIGRvIG5vdCBnZXQKYW55IHJhdGluZ3MuIEFkZGl0aW9uYWxseSwgdGhlIGZhY3QgdGhhdCBtb3N0IFR3aXR0ZXIgdXNlcnMgYXJlIHBhc3NpdmUKY29uc3VtZXJzLCB3aXRoIDQ0XCUgdXNlcnMgbmV2ZXIgdHdlZXRpbmcsIG1ha2VzIGl0IHZlcnkgZGlmZmljdWx0IHRvIHVzZQp1c2VyIHJhdGluZ3MgZm9yIGdlbmVyYXRpbmcgcmVjb21tZW5kYXRpb25zLiBGdXJ0aGVyLCBhIGtleSBjaGFsbGVuZ2UgaW4KZGV2ZWxvcGluZyByZWNvbW1lbmRhdGlvbiBzeXN0ZW1zIGlzIHRoYXQgaW4gbWFueSBjYXNlcyB1c2VycyByZWplY3QKcmVsZXZhbnQgcmVjb21tZW5kYXRpb25zIGlmIHRoZXkgYXJlIHRvdGFsbHkgdW5mYW1pbGlhciB3aXRoIHRoZQpyZWNvbW1lbmRlZCBpdGVtLiBQcm92aWRpbmcgYSBzdWl0YWJsZSBleHBsYW5hdGlvbiwgZm9yIHdoeSB0aGUgaXRlbSBpcwpyZWNvbW1lbmRlZCwgc2lnbmlmaWNhbnRseSBpbXByb3ZlcyB0aGUgYWNjZXB0YWJpbGl0eSBvZiByZWNvbW1lbmRhdGlvbi4KQnkgdmlydHVlIG9mIGJlaW5nIGEgdG9waWNhbCByZWNvbW1lbmRhdGlvbiBzeXN0ZW0gb3VyIG1ldGhvZCBpcyBhYmxlIHRvCnByZXNlbnQgc2ltcGxlIHRvcGljYWwgZXhwbGFuYXRpb25zIGZvciB0aGUgZ2VuZXJhdGVkIHJlY29tbWVuZGF0aW9ucy4KQ29tcGFyaXNvbnMgd2l0aCBzdGF0ZS1vZi10aGUtYXJ0IG1hdHJpeCBmYWN0b3JpemF0aW9uIGJhc2VkIGNvbGxhYm9yYXRpdmUKZmlsdGVyaW5nLCBjb250ZW50IGJhc2VkIGFuZCBzb2NpYWwgcmVjb21tZW5kYXRpb25zIGRlbW9uc3RyYXRlIHRoZQplZmZpY2FjeSBvZiB0aGUgcHJvcG9zZWQgYXBwcm9hY2gufSwKCWF1dGhvciA9IHtCaGF0dGFjaGFyeWEsIFBhcmFudGFwYSBhbmQgR2hvc2gsIFNhcHRhcnNoaSBhbmQgWmFmYXIsIE11aGFtbWFkIEJpbGFsIGFuZCBHaG9zaCwgU291bXlhIEsgYW5kIEdhbmd1bHksIE5pbG95fSwKCW1vbnRoID0gZGVjLAoJeWVhciA9IHsyMDIyfSwKCW5vdGUgPSB7SVNCTjogMjIxMi4xMzg5NwpQdWJsaWNhdGlvbiBUaXRsZTogYXJYaXYgW2NzLklSXX0sCn0KCgoKQHVucHVibGlzaGVke2xvcGVzX3JlY29tbWVuZGF0aW9uc18yMDIyLAoJdGl0bGUgPSB7UmVjb21tZW5kYXRpb25zIHdpdGgge01pbmltdW19IHtFeHBvc3VyZX0ge0d1YXJhbnRlZXN9OiB7QX0ge1Bvc3R9LXtQcm9jZXNzaW5nfSB7RnJhbWV3b3JrfX0sCgl1cmwgPSB7aHR0cHM6Ly9wYXBlcnMuc3Nybi5jb20vYWJzdHJhY3Q9NDI3NDc4MH0sCglhYnN0cmFjdCA9IHtSZWxldmFuY2UtYmFzZWQgcmFua2luZyBpcyBhIHBvcHVsYXIgaW5ncmVkaWVudCBpbiByZWNvbW1lbmRlcnMsIGJ1dCBpdApmcmVxdWVudGx5IHN0cnVnZ2xlcyB0byBtZWV0IGZhaXJuZXNzIGNyaXRlcmlhIGJlY2F1c2Ugc29jaWFsIGFuZCBjdWx0dXJhbApub3JtcyBtYXkgZmF2b3Igc29tZSBpdGVtIGdyb3VwcyBvdmVyIG90aGVycy4gRm9yIGluc3RhbmNlLCBzb21lIGl0ZW1zCm1pZ2h0IHJlY2VpdmUgbG93ZXIgcmF0aW5ncyBkdWUgdG8gc29tZSBzb3J0IG9mIGJpYXMgKGUuZy4gZ2VuZGVyIGJpYXMpLiBBCmZhaXIgcmFua2luZyBzaG91bGQgYmFsYW5jZSB0aGUgZXhwb3N1cmUgb2YgaXRlbXMgZnJvbSBhZHZhbnRhZ2VkIGFuZApkaXNhZHZhbnRhZ2VkZ3JvdXBzLiBUbyB0aGlzIGVuZCwgd2UgcHJvcG9zZSBhIG5vdmVsIHBvc3QtcHJvY2Vzc2luZwpmcmFtZXdvcmsgdG8gcHJvZHVjZSBmYWlyLCBleHBvc3VyZS1hd2FyZSByZWNvbW1lbmRhdGlvbnMuIE91ciBhcHByb2FjaCBpcwpiYXNlZCBvbiBhbiBpbnRlZ2VyIGxpbmVhciBwcm9ncmFtbWluZyBtb2RlbCBtYXhpbWl6aW5nIHRoZSBleHBlY3RlZAp1dGlsaXR5IHdoaWxlIHNhdGlzZnlpbmcgYSBtaW5pbXVtIGV4cG9zdXJlIGNvbnN0cmFpbnQuIFRoZSBtb2RlbCBoYXMKZmV3ZXIgdmFyaWFibGVzIHRoYW4gcHJldmlvdXMgd29yayBhbmQgdGh1cyBjYW4gYmUgZGVwbG95ZWQgdG8gbGFyZ2VyCmRhdGFzZXRzIGFuZCBhbGxvd3MgdGhlIG9yZ2FuaXphdGlvbiB0byBkZWZpbmUgYSBtaW5pbXVtIGxldmVsIG9mIGV4cG9zdXJlCmZvciBncm91cHMgb2YgaXRlbXMuIFdlIGNvbmR1Y3QgYW4gZXh0ZW5zaXZlIGVtcGlyaWNhbCBldmFsdWF0aW9uCmluZGljYXRpbmcgdGhhdCBvdXIgbmV3IGZyYW1ld29yayBjYW4gaW5jcmVhc2UgdGhlIGV4cG9zdXJlIG9mIGl0ZW1zIGZyb20KZGlzYWR2YW50YWdlZCBncm91cHMgYXQgYSBzbWFsbCBjb3N0IG9mIHJlY29tbWVuZGF0aW9uIGFjY3VyYWN5Ln0sCgl1cmxkYXRlID0gezIwMjItMTItMjN9LAoJYXV0aG9yID0ge0xvcGVzLCBSYW1vbiBhbmQgQWx2ZXMsIFJvZHJpZ28gYW5kIExlZGVudCwgQW50b2luZSBhbmQgU2FudG9zLCBSb2RyeWdvIGFuZCBLbG9mdCwgTWFyaXVzfSwKCW1vbnRoID0gbm92LAoJeWVhciA9IHsyMDIyfSwKCWRvaSA9IHsxMC4yMTM5L3Nzcm4uNDI3NDc4MH0sCglrZXl3b3JkcyA9IHtyZWNvbW1lbmRlciBzeXN0ZW1zLCBmYWlybmVzcywgZXhwb3N1cmUsIGludGVnZXIgbGluZWFyIHByb2dyYW1taW5nfSwKfQoKCgpAYXJ0aWNsZXtiYWxjaGFub3dza2lfY29sbGFib3JhdGl2ZV8yMDIyLAoJdGl0bGUgPSB7Q29sbGFib3JhdGl2ZSB7UmFua30ge0FnZ3JlZ2F0aW9ufSBpbiB7UmVjb21tZW5kYXRpb259IHtTeXN0ZW1zfX0sCgl2b2x1bWUgPSB7MjA3fSwKCWlzc24gPSB7MTg3Ny0wNTA5fSwKCXVybCA9IHtodHRwczovL3d3dy5zY2llbmNlZGlyZWN0LmNvbS9zY2llbmNlL2FydGljbGUvcGlpL1MxODc3MDUwOTIyMDExNjdYfSwKCWRvaSA9IHsxMC4xMDE2L2oucHJvY3MuMjAyMi4wOS4yODF9LAoJYWJzdHJhY3QgPSB7T3ZlciB0aGUgeWVhcnMsIHZhcmlvdXMgdGVjaG5pcXVlcyBvZiBnZW5lcmF0aW5nIHJlY29tbWVuZGF0aW9ucyBoYXZlIGJlZW4KZGV2ZWxvcGVkLiBIb3dldmVyLCBpdCB0dXJucyBvdXQgdGhhdCB3aGVuIHdlIGNvbXBhcmUgdGhlIHJlY29tbWVuZGF0aW9ucwpnZW5lcmF0ZWQgYnkgZGlmZmVyZW50IGFsZ29yaXRobXMgaW4gdGhlIGNvbnRleHQgb2YgYSBwYXJ0aWN1bGFyIHVzZXIsIHRoZQpxdWFsaXR5IG9mIHN1Y2ggcmVjb21tZW5kYXRpb25zIGZvciBkaWZmZXJlbnQgdGVjaG5pcXVlcyBtYXkgZGlmZmVyLiBUaGUKdXNlIG9mIHRoZSBhZ2dyZWdhdGlvbiB0ZWNobmlxdWVzLCB0aGUgYWltIG9mIHdoaWNoIGlzIHRvIGNvbWJpbmUgc2V2ZXJhbApyYW5raW5ncyBpbnRvIG9uZSwgY2FuIGJlIGEgc29sdXRpb24gdG8gdGhpcyBwcm9ibGVtLiBJbiB0aGVvcnkgaXQgc2hvdWxkCmltcHJvdmUgdGhlIHF1YWxpdHkgb2YgdGhlIHJlY29tbWVuZGF0aW9ucy4gQWRkaXRpb25hbGx5LCBpbiBvcmRlciB0bwpwZXJzb25hbGl6ZSB0aGUgcmVjb21tZW5kYXRpb25zIGJldHRlciwgYSBtZXRhaGV1cmlzdGljIGFsZ29yaXRobSwgd2hpY2gsCmJ5IGFzc2lnbmluZyBkaWZmZXJlbnQgd2VpZ2h0cyB0byBlYWNoIGZlYXR1cmUsIHRyaWVzIHRvIHJlcHJlc2VudCB0aGUKcHJlZmVyZW5jZSBvZiB0aGUgYWN0aXZlIHVzZXIsIHdhcyB1c2VkLiBUaGlzIHBhcGVyIGFsc28gcHJlc2VudHMgYQpzdWdnZXN0aW9uIHRvIGluY2x1ZGUgYWRkaXRpb25hbCByYW5raW5ncyBnZW5lcmF0ZWQgZm9yIG90aGVyIHVzZXJzIGluIHRoZQpzeXN0ZW0gaW4gdGhlIGFnZ3JlZ2F0aW9uIHByb2Nlc3MuIFRoZSBpZGVhIHdpbGwgYmUgc3VwcG9ydGVkIGJ5IHJlc2VhcmNoCnJlc3VsdHMgdGhhdCBjbGVhcmx5IHNob3cgdGhhdCB0YWtpbmcgaW50byBhY2NvdW50IHJhbmtpbmdzIG9mIG90aGVyIHVzZXJzCmNhbiBpbXByb3ZlIHRoZSBxdWFsaXR5IG9mIHRoZSBnZW5lcmF0ZWQgcmVjb21tZW5kYXRpb25zLn0sCglqb3VybmFsID0ge1Byb2NlZGlhIENvbXB1dC4gU2NpLn0sCglhdXRob3IgPSB7QmHFgmNoYW5vd3NraSwgTWljaGHFgiBhbmQgQm9yeWN6a2EsIFVyc3p1bGF9LAoJbW9udGggPSBqYW4sCgl5ZWFyID0gezIwMjJ9LAoJa2V5d29yZHMgPSB7ZGlmZmVyZW50aWFsIGV2b2x1dGlvbiwgbWV0YWhldXJpc3RpYywgcmFuayBhZ2dyZWdhdGlvbiwgcmVjb21tZW5kYXRpb24gc3lzdGVtc30sCglwYWdlcyA9IHsyMjEzLS0yMjIyfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3t3ZWdtZXRoX2NhbWVsc18yMDIyLAoJdGl0bGUgPSB7e0NhTWVMU306IHtDb29wZXJhdGl2ZX0gbWV0YS1sZWFybmluZyBzZXJ2aWNlIGZvciByZWNvbW1lbmRlciBzeXN0ZW1zfSwKCXVybCA9IHtodHRwczovL2NldXItd3Mub3JnL1ZvbC0zMjI4L3BhcGVyMi5wZGZ9LAoJYWJzdHJhY3QgPSB7V2UgcHJlc2VudCBDYU1lTFMsIGEgcHJvb2Ygb2YgY29uY2VwdCBvZiBhIGNvb3BlcmF0aXZlIG1ldGEtbGVhcm5pbmcKc2VydmljZSBmb3IgcmVjb21tZW5kZXIgc3lzdGVtcy4gQ2FNZUxTIGxldmVyYWdlcyB0aGUgY29tcHV0aW5nIHBvd2VyIG9mCnJlY29tbWVuZGVyIHN5c3RlbXMgdXNlcnMgYnkgdXBsb2FkaW5nIHRoZWlyIG1ldGFkYXRhIGFuZCBhbGdvcml0aG0KZXZhbHVhdGlvbiBzY29yZXMgdG8gYSBjZW50cmFsaXplZCBlbnZpcm9ubWVudC4gVGhyb3VnaCB0aGUgcmVzdWx0aW5nCmRhdGFiYXNlLCBDYU1lTFMgdGhlbiBvZmZlcnMgbWV0YS1sZWFybmluZyBzZXJ2aWNlcyBmb3IgZXZlcnlvbmUuCkFkZGl0aW9uYWxseSwgdXNlcnMgbWF5IGFjY2VzcyBldmFsdWF0aW9ucyBvZiBjb21tb24gZGF0YSBzZXRzIGltbWVkaWF0ZWx5CnRvIGtub3cgdGhlIGJlc3QtcGVyZm9ybWluZyBhbGdvcml0aG1zIGZvciB0aG9zZSBkYXRhIHNldHMuIFRoZSBtZXRhZGF0YQp0YWJsZSBtYXkgYWxzbyBiZSB1c2VkIGZvciBvdGhlciBwdXJwb3NlcywgZS5nLiwgdG8gcGVyZm9ybSBiZW5jaG1hcmtzLiBJbgp0aGUgaW5pdGlhbCB2ZXJzaW9uIGRpc2N1c3NlZCBpbiB0aGlzIHBhcGVyLCBDYU1lTFMgaW1wbGVtZW50cyBhdXRvbWF0aWMKYWxnb3JpdGhtIHNlbGVjdGlvbiB0aHJvdWdoIG1ldGEtbGVhcm5pbmcgb3ZlciB0d28gcmVjb21tZW5kZXIgc3lzdGVtcwpsaWJyYXJpZXMuIEF1dG9tYXRpYyBhbGdvcml0aG0gc2VsZWN0aW9uIHNhdmVzIHVzZXJzIHRpbWUgYW5kIGNvbXB1dGluZwpwb3dlciBhbmQgZG9lcyBub3QgcmVxdWlyZSBleHBlcnRpc2UsIGFzIHRoZSBiZXN0IGFsZ29yaXRobSBpcwphdXRvbWF0aWNhbGx5IGZvdW5kIG92ZXIgbXVsdGlwbGUgbGlicmFyaWVzLiBUaGUgQ2FNZUxTIGRhdGFiYXNlIGNvbnRhaW5zCjIwIG1ldGFkYXRhIHNldHMgYnkgZGVmYXVsdC4gV2Ugc2hvdyB0aGF0IHRoZSBhdXRvbWF0aWMgYWxnb3JpdGhtCnNlbGVjdGlvbiBzZXJ2aWNlIGlzIGFscmVhZHkgb24gcGFyIHdpdGggdGhlIHNpbmdsZSBiZXN0IGFsZ29yaXRobSBpbiB0aGlzCmRlZmF1bHQgc2NlbmFyaW8uIENhTWVMUyBvbmx5IHJlcXVpcmVzIGEgZmV3IHNlY29uZHMgdG8gcHJlZGljdCBhIHN1aXRhYmxlCmFsZ29yaXRobSwgcmF0aGVyIHRoYW4gcG90ZW50aWFsbHkgaG91cnMgb3IgZGF5cyBpZiBwZXJmb3JtZWQgbWFudWFsbHksCmRlcGVuZGluZyBvbiB0aGUgZGF0YSBzZXQuIFRoZSBjb2RlIGlzIHB1YmxpY2x5IGF2YWlsYWJsZSBvbiBvdXIgR2l0SHViCmh0dHBzOi8vY2FtZWxzLnJlY29tbWVuZGVyLXN5c3RlbXMuY29tLn0sCgl1cmxkYXRlID0gezIwMjItMTEtMTN9LAoJcHVibGlzaGVyID0ge0NFVVItV1N9LAoJYXV0aG9yID0ge1dlZ21ldGgsIEx1a2FzIGFuZCBCZWVsLCBKb2VyYW59LAoJbW9udGggPSBzZXAsCgl5ZWFyID0gezIwMjJ9LAp9CgoKCkB1bnB1Ymxpc2hlZHtoYWxwZXJuX3JlcHJlc2VudGF0aW9uXzIwMjIsCgl0aXRsZSA9IHtSZXByZXNlbnRhdGlvbiB3aXRoIHtJbmNvbXBsZXRlfSB7Vm90ZXN9fSwKCXVybCA9IHtodHRwOi8vYXJ4aXYub3JnL2Ficy8yMjExLjE1NjA4fSwKCWFic3RyYWN0ID0ge1BsYXRmb3JtcyBmb3Igb25saW5lIGNpdmljIHBhcnRpY2lwYXRpb24gcmVseSBoZWF2aWx5IG9uIG1ldGhvZHMgZm9yCmNvbmRlbnNpbmcgdGhvdXNhbmRzIG9mIGNvbW1lbnRzIGludG8gYSByZWxldmFudCBoYW5kZnVsLCBiYXNlZCBvbiB3aGV0aGVyCnBhcnRpY2lwYW50cyBhZ3JlZSBvciBkaXNhZ3JlZSB3aXRoIHRoZW0uIFRoZXNlIG1ldGhvZHMgc2hvdWxkIGd1YXJhbnRlZQpmYWlyIHJlcHJlc2VudGF0aW9uIG9mIHRoZSBwYXJ0aWNpcGFudHMsIGFzIHRoZWlyIG91dGNvbWVzIG1heSBhZmZlY3QgdGhlCmhlYWx0aCBvZiB0aGUgY29udmVyc2F0aW9uIGFuZCBpbmZvcm0gaW1wYWN0ZnVsIGRvd25zdHJlYW0gZGVjaXNpb25zLiBUbwp0aGF0IGVuZCwgd2UgZHJhdyBvbiB0aGUgbGl0ZXJhdHVyZSBvbiBhcHByb3ZhbC1iYXNlZCBjb21taXR0ZWUgZWxlY3Rpb25zLgpPdXIgc2V0dGluZyBpcyBub3ZlbCBpbiB0aGF0IHRoZSBhcHByb3ZhbCB2b3RlcyBhcmUgaW5jb21wbGV0ZSBzaW5jZQpwYXJ0aWNpcGFudHMgd2lsbCB0eXBpY2FsbHkgbm90IHZvdGUgb24gYWxsIGNvbW1lbnRzLiBXZSBwcm92ZSB0aGF0IHRoaXMKY29tcGxpY2F0aW9uIHJlbmRlcnMgbm9uLWFkYXB0aXZlIGFsZ29yaXRobXMgaW1wcmFjdGljYWwgaW4gdGVybXMgb2YgdGhlCmFtb3VudCBvZiBpbmZvcm1hdGlvbiB0aGV5IG11c3QgZ2F0aGVyLiBUaGVyZWZvcmUsIHdlIGRldmVsb3AgYW4gYWRhcHRpdmUKYWxnb3JpdGhtIHRoYXQgdXNlcyBpbmZvcm1hdGlvbiBtb3JlIGVmZmljaWVudGx5IGJ5IHByZXNlbnRpbmcgaW5jb21pbmcKcGFydGljaXBhbnRzIHdpdGggc3RhdGVtZW50cyB0aGF0IGFwcGVhciBwcm9taXNpbmcgYmFzZWQgb24gdm90ZXMgYnkKcHJldmlvdXMgcGFydGljaXBhbnRzLiBXZSBwcm92ZSB0aGF0IHRoaXMgbWV0aG9kIHNhdGlzZmllcyBjb21tb25seSB1c2VkCm5vdGlvbnMgb2YgZmFpciByZXByZXNlbnRhdGlvbiwgZXZlbiB3aGVuIHBhcnRpY2lwYW50cyBvbmx5IHZvdGUgb24gYQpzbWFsbCBmcmFjdGlvbiBvZiBjb21tZW50cy4gRmluYWxseSwgYW4gZW1waXJpY2FsIGV2YWx1YXRpb24gdXNpbmcgcmVhbApkYXRhIHNob3dzIHRoYXQgdGhlIHByb3Bvc2VkIGFsZ29yaXRobSBwcm92aWRlcyByZXByZXNlbnRhdGl2ZSBvdXRjb21lcyBpbgpwcmFjdGljZS59LAoJYXV0aG9yID0ge0hhbHBlcm4sIERhbmllbCBhbmQgS2VobmUsIEdyZWdvcnkgYW5kIFByb2NhY2NpYSwgQXJpZWwgRCBhbmQgVHVja2VyLUZvbHR6LCBKYW1pZSBhbmQgV8O8dGhyaWNoLCBNYW51ZWx9LAoJbW9udGggPSBub3YsCgl5ZWFyID0gezIwMjJ9LAoJbm90ZSA9IHtJU0JOOiAyMjExLjE1NjA4ClB1YmxpY2F0aW9uIFRpdGxlOiBhclhpdiBbY3MuR1RdfSwKfQoKCgpAYXJ0aWNsZXtsdV91c2VyXzIwMjIsCgl0aXRsZSA9IHtVc2VyIHtQZXJjZXB0aW9ufSBvZiB7UmVjb21tZW5kYXRpb259IHtFeHBsYW5hdGlvbn06IHtBcmV9IHtZb3VyfSB7RXhwbGFuYXRpb25zfSB7V2hhdH0ge1VzZXJzfSB7TmVlZH0/fSwKCWlzc24gPSB7MTA5NC05MjI0fSwKCXVybCA9IHtodHRwczovL2RvaS5vcmcvMTAuMTE0NS8zNTY1NDgwfSwKCWRvaSA9IHsxMC4xMTQ1LzM1NjU0ODB9LAoJYWJzdHJhY3QgPSB7QXMgcmVjb21tZW5kZXIgc3lzdGVtcyBiZWNvbWUgaW5jcmVhc2luZ2x5IGltcG9ydGFudCBpbiBkYWlseSBodW1hbgpkZWNpc2lvbi1tYWtpbmcsIHVzZXJzIGFyZSBkZW1hbmRpbmcgY29udmluY2luZyBleHBsYW5hdGlvbnMgdG8gdW5kZXJzdGFuZAp0aGV5IGdldCB0aGUgc3BlY2lmaWMgcmVjb21tZW5kYXRpb24gcmVzdWx0cy4gQWx0aG91Z2ggYSBudW1iZXIgb2YKZXhwbGFpbmFibGUgcmVjb21tZW5kZXIgc3lzdGVtcyBoYXZlIHJlY2VudGx5IGJlZW4gcHJvcG9zZWQsIHRoZXJlIHN0aWxsCmxhY2tzIGFuIHVuZGVyc3RhbmRpbmcgb2Ygd2hhdCB1c2VycyByZWFsbHkgbmVlZCBpbiBhIHJlY29tbWVuZGF0aW9uCmV4cGxhbmF0aW9uLiBUaGUgYWN0dWFsIHJlYXNvbiBiZWhpbmQgdXNlcnPigJkgaW50ZW50aW9uIHRvIGV4YW1pbmUgYW5kCmNvbnN1bWUgKGUuZy4sIGNsaWNrIGFuZCB3YXRjaCBhIG1vdmllKSBjYW4gYmUgdGhlIHdpbmRvdyB0byBhbnN3ZXIgdGhpcwpxdWVzdGlvbiBhbmQgaXMgbmFtZWQgYXMgc2VsZi1leHBsYW5hdGlvbiBpbiB0aGlzIHdvcmsuIEluIGFkZGl0aW9uLApodW1hbnMgdXN1YWxseSBtYWtlIHJlY29tbWVuZGF0aW9ucyBhY2NvbXBhbmllZCBieSBleHBsYW5hdGlvbnMsIGJ1dCB0aGVyZQpyZW1haW4gZmV3ZXIgc3R1ZGllcyBvbiBob3cgaHVtYW5zIGV4cGxhaW4gYW5kIHdoYXQgd2UgY2FuIGxlYXJuIGZyb20KaHVtYW4tZ2VuZXJhdGVkIGV4cGxhbmF0aW9ucy4gVG8gaW52ZXN0aWdhdGUgdGhlc2UgcXVlc3Rpb25zLCB3ZSBjb25kdWN0IGEKbm92ZWwgbXVsdGktcm9sZSwgbXVsdGktc2Vzc2lvbiB1c2VyIHN0dWR5IGluIHdoaWNoIHVzZXJzIGludGVyYWN0IHdpdGgKbXVsdGlwbGUgdHlwZXMgb2Ygc3lzdGVtLWdlbmVyYXRlZCBleHBsYW5hdGlvbnMgYXMgd2VsbCBhcyBodW1hbi1nZW5lcmF0ZWQKZXhwbGFuYXRpb25zLCBuYW1lbHkgcGVlci1leHBsYW5hdGlvbi4gRHVyaW5nIHRoZSBzdHVkeSwgdXNlcnPigJkKaW50ZW50aW9ucywgZXhwZWN0YXRpb25zLCBhbmQgZXhwZXJpZW5jZXMgYXJlIHRyYWNrZWQgaW4gc2V2ZXJhbCBwaGFzZXMsCmluY2x1ZGluZyBiZWZvcmUgYW5kIGFmdGVyIHRoZSB1c2VycyBhcmUgcHJlc2VudGVkIHdpdGggYW4gZXhwbGFuYXRpb24gYW5kCmFmdGVyIHRoZSBjb250ZW50IGlzIGV4YW1pbmVkLiBUaHJvdWdoIGNvbXByZWhlbnNpdmUgaW52ZXN0aWdhdGlvbnMsIHRocmVlCm1haW4gZmluZGluZ3MgaGF2ZSBiZWVuIG1hZGU6IEZpcnN0LCB3ZSBvYnNlcnZlIG5vdCBvbmx5IHRoZSBwb3NpdGl2ZSBidXQKYWxzbyB0aGUgbmVnYXRpdmUgZWZmZWN0cyBvZiBleHBsYW5hdGlvbnMsIGFuZCB0aGUgaW1wYWN0IHZhcmllcyBhY3Jvc3MKZGlmZmVyZW50IHR5cGVzIG9mIGV4cGxhbmF0aW9ucy4gTW9yZW92ZXIsIGh1bWFuLWdlbmVyYXRlZCBleHBsYW5hdGlvbiwKcGVlci1leHBsYW5hdGlvbiwgcGVyZm9ybXMgYmV0dGVyIGluIGluY3JlYXNpbmcgdXNlciBpbnRlbnRpb25zIGFuZApoZWxwaW5nIHVzZXJzIHRvIGJldHRlciBjb25zdHJ1Y3QgcHJlZmVyZW5jZXMsIHdoaWNoIHJlc3VsdHMgaW4gYmV0dGVyCnVzZXIgc2F0aXNmYWN0aW9uLiBTZWNvbmQsIGJhc2VkIG9uIHVzZXJz4oCZIHNlbGYtZXhwbGFuYXRpb24sIHRoZQppbmZvcm1hdGlvbiBhY2N1cmFjeSBpcyBtZWFzdXJlZCBhbmQgZm91bmQgdG8gYmUgYSBtYWpvciBmYWN0b3IgYXNzb2NpYXRlZAp3aXRoIHVzZXIgc2F0aXNmYWN0aW9uLiBTb21lIG90aGVyIGZhY3RvcnMsIHN1Y2ggYXMgdW5mYW1pbGlhcml0eSBhbmQKc2ltaWxhcml0eSwgYXJlIGFsc28gZGlzY292ZXJlZCBhbmQgc3VtbWFyaXplZC4gVGhpcmQsIHRocm91Z2ggYW5ub3RhdGlvbnMKb2YgdGhlIGluZm9ybWF0aW9uIGFzcGVjdHMgdXNlZCBpbiB0aGUgaHVtYW4tZ2VuZXJhdGVkIHNlbGYtZXhwbGFuYXRpb24KYW5kIHBlZXItZXhwbGFuYXRpb24sIHBhdHRlcm5zIG9mIGhvdyBodW1hbnMgZXhwbGFpbiBhcmUgaW52ZXN0aWdhdGVkLAppbmNsdWRpbmcgd2hhdCBpbmZvcm1hdGlvbiBhbmQgaG93IG11Y2ggaW5mb3JtYXRpb24gaXMgdXRpbGl6ZWQuIEluCmFkZGl0aW9uLCBiYXNlZCBvbiB0aGUgZmluZGluZ3MsIGEgaHVtYW4taW5zcGlyZWQgZXhwbGFuYXRpb24gYXBwcm9hY2ggaXMKcHJvcG9zZWQgYW5kIGZvdW5kIHRvIGluY3JlYXNlIHVzZXIgc2F0aXNmYWN0aW9uLCByZXZlYWxpbmcgdGhlIHBvdGVudGlhbAppbXByb3ZlbWVudCBvZiBmdXJ0aGVyIGluY29ycG9yYXRpbmcgbW9yZSBodW1hbiBwYXR0ZXJucyBpbiByZWNvbW1lbmRhdGlvbgpleHBsYW5hdGlvbnMuIFRoZXNlIGZpbmRpbmdzIGhhdmUgc2hlZCBsaWdodCBvbiB0aGUgZGVlcGVyIHVuZGVyc3RhbmRpbmcKb2YgdGhlIHJlY29tbWVuZGF0aW9uIGV4cGxhbmF0aW9uIGFuZCBmdXJ0aGVyIHJlc2VhcmNoIG9uIGl0cyBldmFsdWF0aW9uCmFuZCBnZW5lcmF0aW9uLiBGdXJ0aGVybW9yZSwgdGhlIGNvbGxlY3RlZCBkYXRhLCBpbmNsdWRpbmcgaHVtYW4tZ2VuZXJhdGVkCmV4cGxhbmF0aW9ucyBieSBib3RoIHRoZSBleHRlcm5hbCBwZWVycyBhbmQgdGhlIHVzZXJz4oCZIHNlbHZlcywgd2lsbCBiZQpyZWxlYXNlZCB0byBzdXBwb3J0IGZ1dHVyZSByZXNlYXJjaCB3b3JrcyBvbiBleHBsYW5hdGlvbiBldmFsdWF0aW9uLn0sCglqb3VybmFsID0ge0FDTSBUcmFucy4gSW5mLiBTeXN0LiBTZWN1ci59LAoJYXV0aG9yID0ge0x1LCBIb25neXUgYW5kIE1hLCBXZWl6aGkgYW5kIFdhbmcsIFlpZmFuIGFuZCBaaGFuZywgTWluIGFuZCBXYW5nLCBYaWFuZyBhbmQgTGl1LCBZaXF1biBhbmQgQ2h1YSwgVGF0LVNlbmcgYW5kIE1hLCBTaGFvcGluZ30sCgltb250aCA9IG5vdiwKCXllYXIgPSB7MjAyMn0sCglub3RlID0ge1BsYWNlOiBOZXcgWW9yaywgTlksIFVTQQpQdWJsaXNoZXI6IEFzc29jaWF0aW9uIGZvciBDb21wdXRpbmcgTWFjaGluZXJ5fSwKCWtleXdvcmRzID0ge1JlY29tbWVuZGF0aW9uIEV4cGxhbmF0aW9uLCBVc2VyIE1vZGVsaW5nLCBSZWNvbW1lbmRlciBTeXN0ZW19LAp9CgoKCkBhcnRpY2xle2xpdV9uZXdfMjAyMiwKCXRpdGxlID0ge0Ege05ld30ge0NvbGxhYm9yYXRpdmV9IHtGaWx0ZXJpbmd9IHtBbGdvcml0aG19IHtJbnRlZ3JhdGluZ30ge1RpbWV9IGFuZCB7TXVsdGlzaW1pbGFyaXR5fX0sCgl2b2x1bWUgPSB7MjAyMn0sCglpc3NuID0gezEwMjQtMTIzWH0sCgl1cmwgPSB7aHR0cHM6Ly93d3cuaGluZGF3aS5jb20vam91cm5hbHMvbXBlLzIwMjIvMjM0MDY3MS99LAoJZG9pID0gezEwLjExNTUvMjAyMi8yMzQwNjcxfSwKCWFic3RyYWN0ID0ge0FpbWluZyBhdCB0aGUgcHJvYmxlbSBvZiBsb3cgcmVjb21tZW5kYXRpb24gYWNjdXJhY3kgb2YgZXhpc3RpbmcKcmVjb21tZW5kYXRpb24gYWxnb3JpdGhtcywgYW4gYWxnb3JpdGhtIGludGVncmF0aW5nIHRpbWUgZmFjdG9ycyBhbmQKbXVsdGlzaW1pbGFyaXR5IGlzIHByb3Bvc2VkIHRvIGltcHJvdmUgdGhlIGltcGFjdCBvZiBsb25nLXRlcm0gZGF0YSwgdXNlcgphdHRlbnRpb24sIGFuZCBwcm9qZWN0IHBvcHVsYXJpdHkgb24gdGhlIHJlY29tbWVuZGF0aW9uIGFsZ29yaXRobSBhbmQgdGhlCnNpbWlsYXJpdHkgb2YgdXNlciBhdHRyaWJ1dGVzIGlzIGludHJvZHVjZWQgdG8gaW1wcm92ZSB0aGUgcHJvYmxlbSBvZiBjb2xkCnN0YXJ0IHRvIGEgY2VydGFpbiBleHRlbnQuIENvbnNpZGVyaW5nIHRoYXQgdGhlIGxvbmdlciB0aGUgdGltZSwgdGhlIGxlc3MKbGlrZWx5IGl0IGlzIHRvIGJlIHNlbGVjdGVkIGFnYWluLCB0aW1lIGlzIGludHJvZHVjZWQgaW50byB0aGUgYWxnb3JpdGhtCmFzIGEgd2VpZ2h0IGZhY3Rvci4gV2hlbiB0aGUgYmVoYXZpb3Igb2NjdXJzLCBpLmUuLCBpbnRlcmVzdCBpbiB0aGUKcHJvamVjdCwgc28gYXMgdG8ganVkZ2UgdGhlIHNpbWlsYXJpdHkgYmV0d2VlbiB1c2Vycywgbm90IGp1c3QgdGhlIHNjb3JlCnZhbHVlLCB3ZSBub3JtYWxpemUgdGhlIHBvcHVsYXJpdHkgdG8gYXZvaWQgbWlzanVkZ21lbnQgb2YgaGlnaCBzY29yaW5nCmFuZCBwb3B1bGFyIGl0ZW1zLiBCZWNhdXNlIG5ldyB1c2VycyBkbyBub3QgaGF2ZSBwYXN0IHNjb3JlIHJlY29yZHMsIHRoZQpwcm9ibGVtIG9mIGNvbGQgc3RhcnQgY2FuIGJlIHNvbHZlZCBieSBjYWxjdWxhdGluZyB0aGUgc2ltaWxhcml0eSBvZiB1c2VyCmF0dHJpYnV0ZXMuIFRocm91Z2ggdGhlIGNvbXBhcmF0aXZlIGV4cGVyaW1lbnQgb24gTW92aWVsZW5zMTAwSyBkYXRhc2V0CmFuZCBFcGluaW9ucyBkYXRhc2V0LCB0aGUgcmVzdWx0cyBzaG93IHRoYXQgdGhlIGFsZ29yaXRobSBjYW4gaW1wcm92ZSB0aGUKYWNjdXJhY3kgb2YgcmVjb21tZW5kYXRpb24gYW5kIGdpdmUgdXNlcnMgYSBiZXR0ZXIgcmVjb21tZW5kYXRpb24gZWZmZWN0Ln0sCgl1cmxkYXRlID0gezIwMjItMDktMDV9LAoJam91cm5hbCA9IHtNYXRoLiBQcm9ibC4gRW5nLn0sCglhdXRob3IgPSB7TGl1LCBRaW59LAoJbW9udGggPSBhdWcsCgl5ZWFyID0gezIwMjJ9LAoJbm90ZSA9IHtQdWJsaXNoZXI6IEhpbmRhd2l9LAp9CgoKCkB1bnB1Ymxpc2hlZHt3ZWdtZXRoX2ltcGFjdF8yMDIyLAoJdGl0bGUgPSB7VGhlIHtJbXBhY3R9IG9mIHtGZWF0dXJlfSB7UXVhbnRpdHl9IG9uIHtSZWNvbW1lbmRhdGlvbn0ge0FsZ29yaXRobX0ge1BlcmZvcm1hbmNlfToge0F9IHtNb3ZpZWxlbnN9LXsxMDBLfSB7Q2FzZX0ge1N0dWR5fX0sCgl1cmwgPSB7aHR0cDovL2FyeGl2Lm9yZy9hYnMvMjIwNy4wODcxM30sCglhYnN0cmFjdCA9IHtSZWNlbnQgbW9kZWwtYmFzZWQgUmVjb21tZW5kZXIgU3lzdGVtcyAoUmVjU3lzKSBhbGdvcml0aG1zIGVtcGhhc2l6ZSBvbgp0aGUgdXNlIG9mIGZlYXR1cmVzLCBhbHNvIGNhbGxlZCBzaWRlIGluZm9ybWF0aW9uLCBpbiB0aGVpciBkZXNpZ24gc2ltaWxhcgp0byBhbGdvcml0aG1zIGluIE1hY2hpbmUgTGVhcm5pbmcgKE1MKS4gSW4gY29udHJhc3QsIHNvbWUgb2YgdGhlIG1vc3QKcG9wdWxhciBhbmQgdHJhZGl0aW9uYWwgYWxnb3JpdGhtcyBmb3IgUmVjU3lzIHNvbGVseSBmb2N1cyBvbiBhIGdpdmVuCnVzZXItaXRlbS1yYXRpbmcgcmVsYXRpb24gd2l0aG91dCBpbmNsdWRpbmcgc2lkZSBpbmZvcm1hdGlvbi4gVGhlIGdvYWwgb2YKdGhpcyBjYXNlIHN0dWR5IGlzIHRvIHByb3ZpZGUgYSBwZXJmb3JtYW5jZSBjb21wYXJpc29uIGFuZCBhc3Nlc3NtZW50IG9mClJlY1N5cyBhbmQgTUwgYWxnb3JpdGhtcyB3aGVuIHNpZGUgaW5mb3JtYXRpb24gaXMgaW5jbHVkZWQuIFdlIGNob3NlIHRoZQpNb3ZpZWxlbnMtMTAwSyBkYXRhIHNldCBzaW5jZSBpdCBpcyBhIHN0YW5kYXJkIGZvciBjb21wYXJpbmcgUmVjU3lzCmFsZ29yaXRobXMuIFdlIGNvbXBhcmVkIHNpeCBkaWZmZXJlbnQgZmVhdHVyZSBzZXRzIHdpdGggdmFyeWluZyBxdWFudGl0aWVzCm9mIGZlYXR1cmVzIHdoaWNoIHdlcmUgZ2VuZXJhdGVkIGZyb20gdGhlIGJhc2VsaW5lIGRhdGEgYW5kIGV2YWx1YXRlZCBvbiBhCnRvdGFsIG9mIDE5IFJlY1N5cyBhbGdvcml0aG1zLCBiYXNlbGluZSBNTCBhbGdvcml0aG1zLCBBdXRvbWF0ZWQgTWFjaGluZQpMZWFybmluZyAoQXV0b01MKSBwaXBlbGluZXMsIGFuZCBzdGF0ZS1vZi10aGUtYXJ0IFJlY1N5cyBhbGdvcml0aG1zIHRoYXQKaW5jb3Jwb3JhdGUgc2lkZSBpbmZvcm1hdGlvbi4gVGhlIHJlc3VsdHMgc2hvdyB0aGF0IGFkZGl0aW9uYWwgZmVhdHVyZXMKYmVuZWZpdCBhbGwgYWxnb3JpdGhtcyB3ZSBldmFsdWF0ZWQuIEhvd2V2ZXIsIHRoZSBjb3JyZWxhdGlvbiBiZXR3ZWVuCmZlYXR1cmUgcXVhbnRpdHkgYW5kIHBlcmZvcm1hbmNlIGlzIG5vdCBtb25vdG9ub3VzIGZvciBBdXRvTUwgYW5kIFJlY1N5cy4KSW4gdGhlc2UgY2F0ZWdvcmllcywgYW4gYW5hbHlzaXMgb2YgZmVhdHVyZSBpbXBvcnRhbmNlIHJldmVhbGVkIHRoYXQgdGhlCnF1YWxpdHkgb2YgZmVhdHVyZXMgbWF0dGVycyBtb3JlIHRoYW4gcXVhbnRpdHkuIFRocm91Z2hvdXQgb3VyCmV4cGVyaW1lbnRzLCB0aGUgYXZlcmFnZSBwZXJmb3JtYW5jZSBvbiB0aGUgZmVhdHVyZSBzZXQgd2l0aCB0aGUgbG93ZXN0Cm51bWJlciBvZiBmZWF0dXJlcyBpcyBhYm91dCA2XCUgd29yc2UgY29tcGFyZWQgdG8gdGhhdCB3aXRoIHRoZSBoaWdoZXN0IGluCnRlcm1zIG9mIHRoZSBSb290IE1lYW4gU3F1YXJlZCBFcnJvci4gQW4gaW50ZXJlc3Rpbmcgb2JzZXJ2YXRpb24gaXMgdGhhdApBdXRvTUwgb3V0cGVyZm9ybXMgbWF0cml4IGZhY3Rvcml6YXRpb24tYmFzZWQgUmVjU3lzIGFsZ29yaXRobXMgd2hlbgphZGRpdGlvbmFsIGZlYXR1cmVzIGFyZSB1c2VkLiBBbG1vc3QgYWxsIGFsZ29yaXRobXMgdGhhdCBjYW4gaW5jbHVkZSBzaWRlCmluZm9ybWF0aW9uIGhhdmUgaGlnaGVyIHBlcmZvcm1hbmNlIHdoZW4gdXNpbmcgdGhlIGhpZ2hlc3QgcXVhbnRpdHkgb2YKZmVhdHVyZXMuIEluIHRoZSBvdGhlciBjYXNlcywgdGhlIHBlcmZvcm1hbmNlIGRpZmZlcmVuY2UgaXMgbmVnbGlnaWJsZQooe1x0ZXh0bGVzc30xXCUpLiBUaGUgcmVzdWx0cyBzaG93IGEgY2xlYXIgcG9zaXRpdmUgdHJlbmQgZm9yIHRoZSBlZmZlY3Qgb2YgZmVhdHVyZQpxdWFudGl0eSBhcyB3ZWxsIGFzIHRoZSBpbXBvcnRhbnQgZWZmZWN0cyBvZiBmZWF0dXJlIHF1YWxpdHkgb24gdGhlCmV2YWx1YXRlZCBhbGdvcml0aG1zLn0sCglhdXRob3IgPSB7V2VnbWV0aCwgTHVrYXN9LAoJbW9udGggPSBqdWwsCgl5ZWFyID0gezIwMjJ9LAoJbm90ZSA9IHtJU0JOOiAyMjA3LjA4NzEzClB1YmxpY2F0aW9uIFRpdGxlOiBhclhpdiBbY3MuSVJdfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3txdWFuZy1odW5nX2V4cGxvcmluZ18yMDIxLAoJdGl0bGUgPSB7RXhwbG9yaW5nIHtTZXR9LXtJbnNwaXJlZH0ge1NpbWlsYXJpdHl9IHtNZWFzdXJlc30gZm9yIHtDb2xsYWJvcmF0aXZlfSB7RmlsdGVyaW5nfSB7UmVjb21tZW5kYXRpb259fSwKCXVybCA9IHtodHRwczovL2lpc2kuc2lpdC50dS5hYy50aC9LU0UyMDIxL3VwbG9hZHNfZmluYWwvNzlfX2UyOTFjYWY4ZTNhNGNhOGM0N2U5ZWQ2NTdhMWQwZjc2L1tLU0UyMDIxLTAwNzldLUZpbmFsLWNhbWVyYS1yZWFkeS1wYXBlci5wZGZ9LAoJYXV0aG9yID0ge1F1YW5nLUh1bmcsIEwgRSBhbmQgVGhpLVhpbmgsIEwgRX0sCgltb250aCA9IG5vdiwKCXllYXIgPSB7MjAyMX0sCn0KCgoKQGFydGljbGV7c2xva29tX3Rvd2FyZHNfMjAyMSwKCXRpdGxlID0ge1Rvd2FyZHMgdXNlci1vcmllbnRlZCBwcml2YWN5IGZvciByZWNvbW1lbmRlciBzeXN0ZW0gZGF0YToge0F9IHBlcnNvbmFsaXphdGlvbi1iYXNlZCBhcHByb2FjaCB0byBnZW5kZXIgb2JmdXNjYXRpb24gZm9yIHVzZXIgcHJvZmlsZXN9LAoJdm9sdW1lID0gezU4fSwKCWlzc24gPSB7MDMwNi00NTczfSwKCXVybCA9IHtodHRwczovL3d3dy5zY2llbmNlZGlyZWN0LmNvbS9zY2llbmNlL2FydGljbGUvcGlpL1MwMzA2NDU3MzIxMDAyMDY1fSwKCWRvaSA9IHsxMC4xMDE2L2ouaXBtLjIwMjEuMTAyNzIyfSwKCWFic3RyYWN0ID0ge0luIHRoaXMgcGFwZXIsIHdlIHByb3Bvc2UgYSBuZXcgcHJpdmFjeSBzb2x1dGlvbiBmb3IgdGhlIGRhdGEgdXNlZCB0bwp0cmFpbiBhIHJlY29tbWVuZGVyIHN5c3RlbSwgaS5lLiwgdGhlIHVzZXLigJNpdGVtIG1hdHJpeC4gVGhlIHVzZXLigJNpdGVtCm1hdHJpeCBjb250YWlucyBpbXBsaWNpdCBpbmZvcm1hdGlvbiwgd2hpY2ggY2FuIGJlIGluZmVycmVkIHVzaW5nIGEKY2xhc3NpZmllciwgbGVhZGluZyB0byBwb3RlbnRpYWwgcHJpdmFjeSB2aW9sYXRpb25zLiBPdXIgc29sdXRpb24sIGNhbGxlZApQZXJzb25hbGl6ZWQgQmx1cnJpbmcgKFBlckJsdXIpLCBpcyBhIHNpbXBsZSwgeWV0IGVmZmVjdGl2ZSwgYXBwcm9hY2ggdG8KYWRkaW5nIGFuZCByZW1vdmluZyBpdGVtcyBmcm9tIHVzZXJz4oCZIHByb2ZpbGVzIGluIG9yZGVyIHRvIGdlbmVyYXRlIGFuCm9iZnVzY2F0ZWQgdXNlcuKAk2l0ZW0gbWF0cml4LiBUaGUgbm92ZWx0eSBvZiBQZXJCbHVyIGlzIHBlcnNvbmFsaXphdGlvbiBvZgp0aGUgY2hvaWNlIG9mIGl0ZW1zIHVzZWQgZm9yIG9iZnVzY2F0aW9uIHRvIHRoZSBpbmRpdmlkdWFsIHVzZXIgcHJvZmlsZXMuClBlckJsdXIgaXMgZm9ybXVsYXRlZCB3aXRoaW4gYSB1c2VyLW9yaWVudGVkIHBhcmFkaWdtIG9mIHJlY29tbWVuZGVyCnN5c3RlbSBkYXRhIHByaXZhY3kgdGhhdCBhaW1zIGF0IG1ha2luZyBwcml2YWN5IHNvbHV0aW9ucyB1bmRlcnN0YW5kYWJsZSwKdW5vYnRydXNpdmUsIGFuZCB1c2VmdWwgZm9yIHRoZSB1c2VyLiBXaGVuIG9iZnVzY2F0ZWQgZGF0YSBpcyB1c2VkIGZvcgp0cmFpbmluZywgYSByZWNvbW1lbmRlciBzeXN0ZW0gYWxnb3JpdGhtIGlzIGFibGUgdG8gcmVhY2ggcGVyZm9ybWFuY2UKY29tcGFyYWJsZSB0byB3aGF0IGlzIGF0dGFpbmVkIHdoZW4gaXQgaXMgdHJhaW5lZCBvbiB0aGUgb3JpZ2luYWwsCnVub2JmdXNjYXRlZCBkYXRhLiBBdCB0aGUgc2FtZSB0aW1lLCBhIGNsYXNzaWZpZXIgY2FuIG5vIGxvbmdlciByZWxpYWJseQp1c2UgdGhlIG9iZnVzY2F0ZWQgZGF0YSB0byBwcmVkaWN0IHRoZSBnZW5kZXIgb2YgdXNlcnMsIGluZGljYXRpbmcgdGhhdAppbXBsaWNpdCBnZW5kZXIgaW5mb3JtYXRpb24gaGFzIGJlZW4gcmVtb3ZlZC4gSW4gYWRkaXRpb24gdG8gaW50cm9kdWNpbmcKUGVyQmx1ciwgd2UgbWFrZSBzZXZlcmFsIGtleSBjb250cmlidXRpb25zLiBGaXJzdCwgd2UgcHJvcG9zZSBhbgpldmFsdWF0aW9uIHByb3RvY29sIHRoYXQgY3JlYXRlcyBhIGZhaXIgZW52aXJvbm1lbnQgdG8gY29tcGFyZSBiZXR3ZWVuCmRpZmZlcmVudCBvYmZ1c2NhdGlvbiBjb25kaXRpb25zLiBTZWNvbmQsIHdlIGNhcnJ5IG91dCBleHBlcmltZW50cyB0aGF0CnNob3cgdGhhdCBnZW5kZXIgb2JmdXNjYXRpb24gaW1wYWN0cyB0aGUgZmFpcm5lc3MgYW5kIGRpdmVyc2l0eSBvZgpyZWNvbW1lbmRlciBzeXN0ZW0gcmVzdWx0cy4gSW4gc3VtLCBvdXIgd29yayBlc3RhYmxpc2hlcyB0aGF0IGEgc2ltcGxlLAp0cmFuc3BhcmVudCBhcHByb2FjaCB0byBnZW5kZXIgb2JmdXNjYXRpb24gY2FuIHByb3RlY3QgdXNlciBwcml2YWN5IHdoaWxlCmF0IHRoZSBzYW1lIHRpbWUgaW1wcm92aW5nIHJlY29tbWVuZGF0aW9uIHJlc3VsdHMgZm9yIHVzZXJzIGJ5IG1haW50YWluaW5nCmZhaXJuZXNzIGFuZCBlbmhhbmNpbmcgZGl2ZXJzaXR5Ln0sCgludW1iZXIgPSB7Nn0sCglqb3VybmFsID0ge0luZi4gUHJvY2Vzcy4gTWFuYWcufSwKCWF1dGhvciA9IHtTbG9rb20sIE1hbmVsIGFuZCBIYW5qYWxpYywgQWxhbiBhbmQgTGFyc29uLCBNYXJ0aGF9LAoJbW9udGggPSBub3YsCgl5ZWFyID0gezIwMjF9LAoJa2V5d29yZHMgPSB7RGl2ZXJzaXR5LCBFdmFsdWF0aW9uLCBGYWlybmVzcywgR2VuZGVyIGluZmVyZW5jZSwgT2JmdXNjYXRpb24sIFByaXZhY3ksIFRvcC1OIHJlY29tbWVuZGF0aW9ufSwKCXBhZ2VzID0gezEwMjcyMn0sCn0KCgoKQHVucHVibGlzaGVke21hb19hcHBsaWNhdGlvbl8yMDIxLAoJdGl0bGUgPSB7QXBwbGljYXRpb24gb2Yge0tub3dsZWRnZX0ge0dyYXBoc30gdG8ge1Byb3ZpZGV9IHtTaWRlfSB7SW5mb3JtYXRpb259IGZvciB7SW1wcm92ZWR9IHtSZWNvbW1lbmRhdGlvbn0ge0FjY3VyYWN5fX0sCgl1cmwgPSB7aHR0cDovL2FyeGl2Lm9yZy9hYnMvMjEwMS4wMzA1NH0sCglhYnN0cmFjdCA9IHtQZXJzb25hbGl6ZWQgcmVjb21tZW5kYXRpb25zIGFyZSBwb3B1bGFyIGluIHRoZXNlIGRheXMgb2YgSW50ZXJuZXQgZHJpdmVuCmFjdGl2aXRpZXMsIHNwZWNpZmljYWxseSBzaG9wcGluZy4gUmVjb21tZW5kYXRpb24gbWV0aG9kcyBjYW4gYmUgZ3JvdXBlZAppbnRvIHRocmVlIG1ham9yIGNhdGVnb3JpZXMsIGNvbnRlbnQgYmFzZWQgZmlsdGVyaW5nLCBjb2xsYWJvcmF0aXZlCmZpbHRlcmluZyBhbmQgbWFjaGluZSBsZWFybmluZyBlbmhhbmNlZC4gSW5mb3JtYXRpb24gYWJvdXQgcHJvZHVjdHMgYW5kCnByZWZlcmVuY2VzIG9mIGRpZmZlcmVudCB1c2VycyBhcmUgcHJpbWFyaWx5IHVzZWQgdG8gaW5mZXIgcHJlZmVyZW5jZXMgZm9yCmEgc3BlY2lmaWMgdXNlci4gSW5hZGVxdWF0ZSBpbmZvcm1hdGlvbiBjYW4gb2J2aW91c2x5IGNhdXNlIHRoZXNlIG1ldGhvZHMKdG8gZmFpbCBvciBwZXJmb3JtIHBvb3JseS4gVGhlIG1vcmUgaW5mb3JtYXRpb24gd2UgcHJvdmlkZSB0byB0aGVzZQptZXRob2RzLCB0aGUgbW9yZSBsaWtlbHkgaXQgaXMgdGhhdCB0aGUgbWV0aG9kcyBwZXJmb3JtIGJldHRlci4gS25vd2xlZGdlCmdyYXBocyByZXByZXNlbnQgdGhlIGN1cnJlbnQgdHJlbmQgaW4gcmVjb3JkaW5nIGluZm9ybWF0aW9uIGluIHRoZSBmb3JtIG9mCnJlbGF0aW9ucyBiZXR3ZWVuIGVudGl0aWVzLCBhbmQgY2FuIHByb3ZpZGUgYWRkaXRpb25hbCAoc2lkZSkgaW5mb3JtYXRpb24KYWJvdXQgcHJvZHVjdHMgYW5kIHVzZXJzLiBTdWNoIGluZm9ybWF0aW9uIGNhbiBiZSB1c2VkIHRvIGltcHJvdmUgbmVhcmVzdApuZWlnaGJvdXIgc2VhcmNoLCBjbHVzdGVyaW5nIHVzZXJzIGFuZCBwcm9kdWN0cywgb3IgdHJhaW4gdGhlIG5ldXJhbApuZXR3b3JrLCB3aGVuIG9uZSBpcyB1c2VkLiBJbiB0aGlzIHdvcmssIHdlIHByZXNlbnQgYSBuZXcgZ2VuZXJpYwpyZWNvbW1lbmRhdGlvbiBzeXN0ZW1zIGZyYW1ld29yaywgdGhhdCBpbnRlZ3JhdGVzIGtub3dsZWRnZSBncmFwaHMgaW50bwp0aGUgcmVjb21tZW5kYXRpb24gcGlwZWxpbmUuIFdlIGRlc2NyaWJlIGl0cyBzb2Z0d2FyZSBkZXNpZ24gYW5kCmltcGxlbWVudGF0aW9uLCBhbmQgdGhlbiBzaG93IHRocm91Z2ggZXhwZXJpbWVudHMsIGhvdyBzdWNoIGEgZnJhbWV3b3JrCmNhbiBiZSBzcGVjaWFsaXplZCBmb3IgYSBkb21haW4sIHNheSBtb3ZpZSByZWNvbW1lbmRhdGlvbnMsIGFuZCB0aGUKaW1wcm92ZW1lbnRzIGluIHJlY29tbWVuZGF0aW9uIHJlc3VsdHMgcG9zc2libGUgZHVlIHRvIHNpZGUgaW5mb3JtYXRpb24Kb2J0YWluZWQgZnJvbSBrbm93bGVkZ2UgZ3JhcGhzIHJlcHJlc2VudGF0aW9uIG9mIHN1Y2ggaW5mb3JtYXRpb24uIE91cgpmcmFtZXdvcmsgc3VwcG9ydHMgZGlmZmVyZW50IGtub3dsZWRnZSBncmFwaCByZXByZXNlbnRhdGlvbiBmb3JtYXRzLCBhbmQKZmFjaWxpdGF0ZXMgZm9ybWF0IGNvbnZlcnNpb24sIG1lcmdpbmcgYW5kIGluZm9ybWF0aW9uIGV4dHJhY3Rpb24gbmVlZGVkCmZvciB0cmFpbmluZyByZWNvbW1lbmRhdGlvbiBtZXRob2RzLn0sCglhdXRob3IgPSB7TWFvLCBZdWhhbyBhbmQgTW9raG92LCBTZXJndWVpIEEgYW5kIE11ZHVyLCBTdWRoaXIgUH0sCgltb250aCA9IGphbiwKCXllYXIgPSB7MjAyMX0sCglkb2kgPSB7MTAuMTAwNy9zMTEyNTctMDE4LTkyMTMteH0sCglub3RlID0ge0lTQk46IDIxMDEuMDMwNTQKUHVibGljYXRpb24gVGl0bGU6IGFyWGl2IFtjcy5JUl19LAp9CgoKCkBwaGR0aGVzaXN7Zm9ydGVzX2VuaGFuY2luZ18yMDIyLAoJdGl0bGUgPSB7RW5oYW5jaW5nIHRoZSB7TXVsdGl9LXtPYmplY3RpdmV9IHtSZWNvbW1lbmRhdGlvbn0gZnJvbSB0aHJlZSBuZXcgcGVyc3BlY3RpdmVzOiBkYXRhIGNoYXJhY3Rlcml6YXRpb24sIHJpc2stc2Vuc2l0aXZlbmVzcywgYW5kIHByaW9yaXRpemF0aW9uIG9mIHRoZSBvYmplY3RpdmVzfSwKCXVybCA9IHtodHRwczovL3JlcG9zaXRvcmlvLnVmbWcuYnIvYml0c3RyZWFtLzE4NDMvNDM5MTUvMS9SZWluYWxkb19TaWx2YV9Gb3J0ZXNfVGVzZV9VRk1HLnBkZn0sCglzY2hvb2wgPSB7VW5pdmVyc2lkYWRlIEZlZGVyYWwgZGUgTWluYXMgR2VyYWlzfSwKCWF1dGhvciA9IHtGb3J0ZXMsIFJlbmFsZG8gU2lsdmF9LAoJY29sbGFib3JhdG9yID0ge0dvbsOnYWx2ZXMsIE1hcmNvcyBBbmRyw6l9LAoJeWVhciA9IHsyMDIyfSwKCW5vdGUgPSB7UHVibGljYXRpb24gVGl0bGU6IENpw6puY2lhIGRhIENvbXB1dGHDp8OjbwpWb2x1bWU6IERvY3RvcmFsfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tzY2hlaWR0X3RpbWUtZGVwZW5kZW50XzIwMjEsCgl0aXRsZSA9IHtUaW1lLWRlcGVuZGVudCBldmFsdWF0aW9uIG9mIHJlY29tbWVuZGVyIHN5c3RlbXN9LAoJdm9sdW1lID0gezI5NTV9LAoJdXJsID0ge2h0dHBzOi8vcGVyc3BlY3RpdmVzLXdzLmdpdGh1Yi5pby8yMDIxL3Byb2NlZWRpbmdzL3BhcGVyMTAucGRmfSwKCXVybGRhdGUgPSB7MjAyMS0wOS0yN30sCglib29rdGl0bGUgPSB7e0NFVVJ9LXtXU319LAoJYXV0aG9yID0ge1NjaGVpZHQsIFRlcmVzYSBhbmQgQmVlbCwgSm9lcmFufSwKCW1vbnRoID0gc2VwLAoJeWVhciA9IHsyMDIxfSwKCW5vdGUgPSB7Sm91cm5hbCBBYmJyZXZpYXRpb246IENFVVItV1N9LAp9CgoKCkBpbnByb2NlZWRpbmdze211ZHVyX2ZyYW1ld29ya18yMDIxLAoJYWRkcmVzcyA9IHtOZXcgWW9yaywgTlksIFVTQX0sCgl0aXRsZSA9IHtBIHtGcmFtZXdvcmt9IGZvciB7RW5oYW5jaW5nfSB7RGVlcH0ge0xlYXJuaW5nfSB7QmFzZWR9IHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN9IHdpdGgge0tub3dsZWRnZX0ge0dyYXBoc319LAoJdXJsID0ge2h0dHBzOi8vZG9pLm9yZy8xMC4xMTQ1LzM0NzIxNjMuMzQ3MjE4M30sCglkb2kgPSB7MTAuMTE0NS8zNDcyMTYzLjM0NzIxODN9LAoJYWJzdHJhY3QgPSB7UmVjb21tZW5kYXRpb24gbWV0aG9kcyBmYWxsIGludG8gdGhyZWUgbWFqb3IgY2F0ZWdvcmllcywgY29udGVudCBiYXNlZApmaWx0ZXJpbmcsIGNvbGxhYm9yYXRpdmUgZmlsdGVyaW5nIGFuZCBkZWVwIGxlYXJuaW5nIGJhc2VkLiBJbmZvcm1hdGlvbgphYm91dCBwcm9kdWN0cyBhbmQgdGhlIHByZWZlcmVuY2VzIG9mIGVhcmxpZXIgdXNlcnMgYXJlIHVzZWQgaW4gYW4KdW5zdXBlcnZpc2VkIG1hbm5lciB0byBjcmVhdGUgbW9kZWxzIHdoaWNoIGhlbHAgbWFrZSBwZXJzb25hbGl6ZWQKcmVjb21tZW5kYXRpb25zIHRvIGEgc3BlY2lmaWMgbmV3IHVzZXIuIFRoZSBtb3JlIGluZm9ybWF0aW9uIHdlIHByb3ZpZGUgdG8KdGhlc2UgbWV0aG9kcywgdGhlIG1vcmUgbGlrZWx5IGl0IGlzIHRoYXQgdGhleSB5aWVsZCBiZXR0ZXIKcmVjb21tZW5kYXRpb25zLiBEZWVwIGxlYXJuaW5nIGJhc2VkIG1ldGhvZHMgYXJlIHJlbGF0aXZlbHkgcmVjZW50LCBhbmQKYXJlIGdlbmVyYWxseSBtb3JlIHJvYnVzdCB0byBub2lzZSBhbmQgbWlzc2luZyBpbmZvcm1hdGlvbi4gVGhpcyBpcwpiZWNhdXNlIGRlZXAgbGVhcm5pbmcgbW9kZWxzIGNhbiBiZSB0cmFpbmVkIGV2ZW4gd2hlbiBzb21lIG9mIHRoZQppbmZvcm1hdGlvbiByZWNvcmRzIGhhdmUgcGFydGlhbCBpbmZvcm1hdGlvbi4gS25vd2xlZGdlIGdyYXBocyByZXByZXNlbnQKdGhlIGN1cnJlbnQgdHJlbmQgaW4gcmVjb3JkaW5nIGluZm9ybWF0aW9uIGluIHRoZSBmb3JtIG9mIHJlbGF0aW9ucwpiZXR3ZWVuIGVudGl0aWVzLCBhbmQgY2FuIHByb3ZpZGUgYW55IGF2YWlsYWJsZSBpbmZvcm1hdGlvbiBhYm91dCBwcm9kdWN0cwphbmQgdXNlcnMuIFRoaXMgaW5mb3JtYXRpb24gaXMgdXNlZCB0byB0cmFpbiB0aGUgcmVjb21tZW5kYXRpb24gbW9kZWwuIEluCnRoaXMgd29yaywgd2UgcHJlc2VudCBhIG5ldyBnZW5lcmljIHJlY29tbWVuZGVyIHN5c3RlbXMgZnJhbWV3b3JrLCB0aGF0CmludGVncmF0ZXMga25vd2xlZGdlIGdyYXBocyBpbnRvIHRoZSByZWNvbW1lbmRhdGlvbiBwaXBlbGluZS4gV2UgZGVzY3JpYmUKaXRzIGRlc2lnbiBhbmQgaW1wbGVtZW50YXRpb24sIGFuZCB0aGVuIHNob3cgdGhyb3VnaCBleHBlcmltZW50cywgaG93IHN1Y2gKYSBmcmFtZXdvcmsgY2FuIGJlIHNwZWNpYWxpemVkLCB0YWtpbmcgdGhlIGRvbWFpbiBvZiBtb3ZpZXMgYXMgYW4gZXhhbXBsZSwKYW5kIHRoZSByZXN1bHRpbmcgaW1wcm92ZW1lbnRzIGluIHJlY29tbWVuZGF0aW9ucyBtYWRlIHBvc3NpYmxlIGJ5IHVzaW5nCmFsbCB0aGUgaW5mb3JtYXRpb24gb2J0YWluZWQgdXNpbmcga25vd2xlZGdlIGdyYXBocy4gT3VyIGZyYW1ld29yaywgdG8gYmUKbWFkZSBwdWJsaWNseSBhdmFpbGFibGUsIHN1cHBvcnRzIGRpZmZlcmVudCBrbm93bGVkZ2UgZ3JhcGggcmVwcmVzZW50YXRpb24KZm9ybWF0cywgYW5kIGZhY2lsaXRhdGVzIGZvcm1hdCBjb252ZXJzaW9uLCBtZXJnaW5nIGFuZCBpbmZvcm1hdGlvbgpleHRyYWN0aW9uIG5lZWRlZCBmb3IgdHJhaW5pbmcgcmVjb21tZW5kYXRpb24gbW9kZWxzLn0sCgl1cmxkYXRlID0gezIwMjEtMDktMTR9LAoJYm9va3RpdGxlID0ge3tJREVBU30gMjAyMX0sCglwdWJsaXNoZXIgPSB7QXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAoJYXV0aG9yID0ge011ZHVyLCBTdWRoaXIgUCBhbmQgTW9raG92LCBTZXJndWVpIEEgYW5kIE1hbywgWXVoYW99LAoJbW9udGggPSBqdWwsCgl5ZWFyID0gezIwMjF9LAoJbm90ZSA9IHtKb3VybmFsIEFiYnJldmlhdGlvbjogSURFQVMgMjAyMX0sCglrZXl3b3JkcyA9IHtmcmFtZXdvcmssIGtub3dsZWRnZSBncmFwaHMsIHJlY29tbWVuZGF0aW9uIG1vZGVsIHRyYWluaW5nLCByZWNvbW1lbmRlciBzeXN0ZW0sIGRlZXAgbGVhcm5pbmcgYmFzZWQgcmVjb21tZW5kYXRpb25zfSwKCXBhZ2VzID0gezExLS0yMH0sCn0KCgoKQHVucHVibGlzaGVke2x1Y2hlcmluaV90LXJlY3NfMjAyMSwKCXRpdGxlID0ge1Qte1JFQ1N9OiB7QX0gc2ltdWxhdGlvbiB0b29sIHRvIHN0dWR5IHRoZSBzb2NpZXRhbCBpbXBhY3Qgb2YgcmVjb21tZW5kZXIgc3lzdGVtc30sCgl1cmwgPSB7aHR0cDovL2FyeGl2Lm9yZy9hYnMvMjEwNy4wODk1OX0sCglhYnN0cmFjdCA9IHtTaW11bGF0aW9uIGhhcyBlbWVyZ2VkIGFzIGEgcG9wdWxhciBtZXRob2QgdG8gc3R1ZHkgdGhlIGxvbmctdGVybSBzb2NpZXRhbApjb25zZXF1ZW5jZXMgb2YgcmVjb21tZW5kZXIgc3lzdGVtcy4gVGhpcyBhcHByb2FjaCBhbGxvd3MgcmVzZWFyY2hlcnMgdG8Kc3BlY2lmeSB0aGVpciB0aGVvcmV0aWNhbCBtb2RlbCBleHBsaWNpdGx5IGFuZCBvYnNlcnZlIHRoZSBldm9sdXRpb24gb2YKc3lzdGVtLWxldmVsIG91dGNvbWVzIG92ZXIgdGltZS4gSG93ZXZlciwgcGVyZm9ybWluZyBzaW11bGF0aW9uLWJhc2VkCnN0dWRpZXMgb2Z0ZW4gcmVxdWlyZXMgcmVzZWFyY2hlcnMgdG8gYnVpbGQgdGhlaXIgb3duIHNpbXVsYXRpb24KZW52aXJvbm1lbnRzIGZyb20gdGhlIGdyb3VuZCB1cCwgd2hpY2ggY3JlYXRlcyBhIGhpZ2ggYmFycmllciB0byBlbnRyeSwKaW50cm9kdWNlcyByb29tIGZvciBpbXBsZW1lbnRhdGlvbiBlcnJvciwgYW5kIG1ha2VzIGl0IGRpZmZpY3VsdCB0bwpkaXNlbnRhbmdsZSB3aGV0aGVyIG9ic2VydmVkIG91dGNvbWVzIGFyZSBkdWUgdG8gdGhlIG1vZGVsIG9yIHRoZQppbXBsZW1lbnRhdGlvbi4gV2UgaW50cm9kdWNlIFQtUkVDUywgYW4gb3Blbi1zb3VyY2VkIFB5dGhvbiBwYWNrYWdlCmRlc2lnbmVkIGZvciByZXNlYXJjaGVycyB0byBzaW11bGF0ZSByZWNvbW1lbmRhdGlvbiBzeXN0ZW1zIGFuZCBvdGhlcgp0eXBlcyBvZiBzb2Npb3RlY2huaWNhbCBzeXN0ZW1zIGluIHdoaWNoIGFuIGFsZ29yaXRobSBtZWRpYXRlcyB0aGUKaW50ZXJhY3Rpb25zIGJldHdlZW4gbXVsdGlwbGUgc3Rha2Vob2xkZXJzLCBzdWNoIGFzIHVzZXJzIGFuZCBjb250ZW50CmNyZWF0b3JzLiBUbyBkZW1vbnN0cmF0ZSB0aGUgZmxleGliaWxpdHkgb2YgVC1SRUNTLCB3ZSBwZXJmb3JtIGEKcmVwbGljYXRpb24gb2YgdHdvIHByaW9yIHNpbXVsYXRpb24tYmFzZWQgcmVzZWFyY2ggb24gc29jaW90ZWNobmljYWwKc3lzdGVtcy4gV2UgYWRkaXRpb25hbGx5IHNob3cgaG93IFQtUkVDUyBjYW4gYmUgdXNlZCB0byBnZW5lcmF0ZSBub3ZlbAppbnNpZ2h0cyB3aXRoIG1pbmltYWwgb3ZlcmhlYWQuIE91ciB0b29sIHByb21vdGVzIHJlcHJvZHVjaWJpbGl0eSBpbiB0aGlzCmFyZWEgb2YgcmVzZWFyY2gsIHByb3ZpZGVzIGEgdW5pZmllZCBsYW5ndWFnZSBmb3Igc2ltdWxhdGluZwpzb2Npb3RlY2huaWNhbCBzeXN0ZW1zLCBhbmQgcmVtb3ZlcyB0aGUgZnJpY3Rpb24gb2YgaW1wbGVtZW50aW5nCnNpbXVsYXRpb25zIGZyb20gc2NyYXRjaC59LAoJYXV0aG9yID0ge0x1Y2hlcmluaSwgRWxpIGFuZCBTdW4sIE1hdHRoZXcgYW5kIFdpbmVjb2ZmLCBBbXkgYW5kIE5hcmF5YW5hbiwgQXJ2aW5kfSwKCW1vbnRoID0ganVsLAoJeWVhciA9IHsyMDIxfSwKCW5vdGUgPSB7SVNCTjogMjEwNy4wODk1OQpQdWJsaWNhdGlvbiBUaXRsZTogYXJYaXYgW2NzLkNZXX0sCn0KCgoKQGlucHJvY2VlZGluZ3N7bGlfbmV3XzIwMjEsCglhZGRyZXNzID0ge05ldyBZb3JrLCBOWSwgVVNBfSwKCXRpdGxlID0ge05ldyB7SW5zaWdodHN9IGludG8ge01ldHJpY30ge09wdGltaXphdGlvbn0gZm9yIHtSYW5raW5nfS1iYXNlZCB7UmVjb21tZW5kYXRpb259fSwKCXVybCA9IHtodHRwczovL2RvaS5vcmcvMTAuMTE0NS8zNDA0ODM1LjM0NjI5NzN9LAoJZG9pID0gezEwLjExNDUvMzQwNDgzNS4zNDYyOTczfSwKCWFic3RyYWN0ID0ge0RpcmVjdCBvcHRpbWl6YXRpb24gb2YgSVIgbWV0cmljcyBoYXMgb2Z0ZW4gYmVlbiBhZG9wdGVkIGFzIGFuIGFwcHJvYWNoIHRvCmRldmlzZSBhbmQgZGV2ZWxvcCByYW5raW5nLWJhc2VkIHJlY29tbWVuZGVyIHN5c3RlbXMuIE1vc3QgbWV0aG9kcwpmb2xsb3dpbmcgdGhpcyBhcHByb2FjaCAoZS5nLiBURk1BUCwgQ0xpTUYsIFRvcC1OLVJhbmspIGFpbSBhdCBvcHRpbWl6aW5nCnRoZSBzYW1lIG1ldHJpYyBiZWluZyB1c2VkIGZvciBldmFsdWF0aW9uLCB1bmRlciB0aGUgYXNzdW1wdGlvbiB0aGF0IHRoaXMKd2lsbCBsZWFkIHRvIHRoZSBiZXN0IHBlcmZvcm1hbmNlLiBBIG51bWJlciBvZiBzdHVkaWVzIG9mIHRoaXMgcHJhY3RpY2UKYnJpbmcgdGhpcyBhc3N1bXB0aW9uLCBob3dldmVyLCBpbnRvIHF1ZXN0aW9uLiBJbiB0aGlzIHBhcGVyLCB3ZSBkaWcKZGVlcGVyIGludG8gdGhpcyBpc3N1ZSBpbiBvcmRlciB0byBsZWFybiBtb3JlIGFib3V0IHRoZSBlZmZlY3RzIG9mIHRoZQpjaG9pY2Ugb2YgdGhlIG1ldHJpYyB0byBvcHRpbWl6ZSBvbiB0aGUgcGVyZm9ybWFuY2Ugb2YgYSByYW5raW5nLWJhc2VkCnJlY29tbWVuZGVyIHN5c3RlbS4gV2UgcHJlc2VudCBhbiBleHRlbnNpdmUgZXhwZXJpbWVudGFsIHN0dWR5IGNvbmR1Y3RlZApvbiBkaWZmZXJlbnQgZGF0YXNldHMgaW4gYm90aCBwYWlyd2lzZSBhbmQgbGlzdHdpc2UgbGVhcm5pbmctdG8tcmFuayAoTFRSKQpzY2VuYXJpb3MsIHRvIGNvbXBhcmUgdGhlIHJlbGF0aXZlIG1lcml0IG9mIGZvdXIgcG9wdWxhciBJUiBtZXRyaWNzLApuYW1lbHkgUlIsIEFQLCBuRENHIGFuZCBSQlAsIHdoZW4gdXNlZCBmb3Igb3B0aW1pemF0aW9uIGFuZCBhc3Nlc3NtZW50IG9mCnJlY29tbWVuZGVyIHN5c3RlbXMgaW4gdmFyaW91cyBjb21iaW5hdGlvbnMuIEZvciB0aGUgZmlyc3QgdGhyZWUsIHdlCmZvbGxvdyB0aGUgcHJhY3RpY2Ugb2YgbG9zcyBmdW5jdGlvbiBmb3JtdWxhdGlvbiBhdmFpbGFibGUgaW4gbGl0ZXJhdHVyZS4KRm9yIHRoZSBmb3VydGggb25lLCB3ZSBwcm9wb3NlIG5vdmVsIGxvc3MgZnVuY3Rpb25zIGluc3BpcmVkIGJ5IFJCUCBmb3IKYm90aCB0aGUgcGFpcndpc2UgYW5kIGxpc3R3aXNlIHNjZW5hcmlvLiBPdXIgcmVzdWx0cyBjb25maXJtIHRoYXQgdGhlIGJlc3QKcGVyZm9ybWFuY2UgaXMgaW5kZWVkIG5vdCBuZWNlc3NhcmlseSBhY2hpZXZlZCB3aGVuIG9wdGltaXppbmcgdGhlIHNhbWUKbWV0cmljIGJlaW5nIHVzZWQgZm9yIGV2YWx1YXRpb24uIEluIGZhY3QsIHdlIGZpbmQgdGhhdCBSQlAtaW5zcGlyZWQKbG9zc2VzIHBlcmZvcm0gYXQgbGVhc3QgYXMgd2VsbCBhcyBvdGhlciBtZXRyaWNzIGluIGEgY29uc2lzdGVudCB3YXksIGFuZApvZmZlciBjbGVhciBiZW5lZml0cyBpbiBzZXZlcmFsIGNhc2VzLiBJbnRlcmVzdGluZyB0byBzZWUgaXMgdGhhdApSQlAtaW5zcGlyZWQgbG9zc2VzLCB3aGlsZSBpbXByb3ZpbmcgdGhlIHJlY29tbWVuZGF0aW9uIHBlcmZvcm1hbmNlIGZvcgphbGwgdXNlcywgbWF5IGxlYWQgdG8gYW4gaW5kaXZpZHVhbCBwZXJmb3JtYW5jZSBnYWluIHRoYXQgaXMgY29ycmVsYXRlZAp3aXRoIHRoZSBhY3Rpdml0eSBsZXZlbCBvZiBhIHVzZXIgaW4gaW50ZXJhY3Rpbmcgd2l0aCBpdGVtcy4gVGhlIG1vcmUKYWN0aXZlIHRoZSB1c2VycywgdGhlIG1vcmUgdGhleSBiZW5lZml0LiBPdmVyYWxsLCBvdXIgcmVzdWx0cyBjaGFsbGVuZ2UKdGhlIGFzc3VtcHRpb24gYmVoaW5kIHRoZSBjdXJyZW50IHJlc2VhcmNoIHByYWN0aWNlIG9mIG9wdGltaXppbmcgYW5kCmV2YWx1YXRpbmcgdGhlIHNhbWUgbWV0cmljLCBhbmQgcG9pbnQgdG8gUkJQLWJhc2VkIG9wdGltaXphdGlvbiBpbnN0ZWFkIGFzCmEgcHJvbWlzaW5nIGFsdGVybmF0aXZlIHdoZW4gbGVhcm5pbmcgdG8gcmFuayBpbiB0aGUgcmVjb21tZW5kYXRpb24KY29udGV4dC59LAoJdXJsZGF0ZSA9IHsyMDIxLTA3LTE1fSwKCWJvb2t0aXRsZSA9IHt7U0lHSVJ9ICcyMX0sCglwdWJsaXNoZXIgPSB7QXNzb2NpYXRpb24gZm9yIENvbXB1dGluZyBNYWNoaW5lcnl9LAoJYXV0aG9yID0ge0xpLCBSb2dlciBaaGUgYW5kIFVyYmFubywgSnVsacOhbiBhbmQgSGFuamFsaWMsIEFsYW59LAoJbW9udGggPSBqdWwsCgl5ZWFyID0gezIwMjF9LAoJbm90ZSA9IHtKb3VybmFsIEFiYnJldmlhdGlvbjogU0lHSVIgJzIxfSwKCWtleXdvcmRzID0ge2V2YWx1YXRpb24gbWV0cmljcywgbGVhcm5pbmcgdG8gcmFuaywgcmVjb21tZW5kZXIgc3lzdGVtc30sCglwYWdlcyA9IHs5MzItLTk0MX0sCn0KCgoKQGFydGljbGV7YXNob2thbl9mYWlybmVzc18yMDIxLAoJdGl0bGUgPSB7RmFpcm5lc3MgbWV0cmljcyBhbmQgYmlhcyBtaXRpZ2F0aW9uIHN0cmF0ZWdpZXMgZm9yIHJhdGluZyBwcmVkaWN0aW9uc30sCgl2b2x1bWUgPSB7NTh9LAoJaXNzbiA9IHswMzA2LTQ1NzN9LAoJdXJsID0ge2h0dHBzOi8vd3d3LnNjaWVuY2VkaXJlY3QuY29tL3NjaWVuY2UvYXJ0aWNsZS9waWkvUzAzMDY0NTczMjEwMDEzNjl9LAoJZG9pID0gezEwLjEwMTYvai5pcG0uMjAyMS4xMDI2NDZ9LAoJYWJzdHJhY3QgPSB7QWxnb3JpdGhtIGZhaXJuZXNzIGlzIGFuIGVzdGFibGlzaGVkIGxpbmUgb2YgcmVzZWFyY2ggaW4gdGhlIG1hY2hpbmUKbGVhcm5pbmcgZG9tYWluIHdpdGggc3Vic3RhbnRpYWwgd29yayB3aGlsZSB0aGUgZXF1aXZhbGVudCBpbiB0aGUKcmVjb21tZW5kZXIgc3lzdGVtIGRvbWFpbiBpcyByZWxhdGl2ZWx5IG5ldy4gSW4gdGhpcyBhcnRpY2xlLCB3ZSBjb25zaWRlcgpyYXRpbmctYmFzZWQgcmVjb21tZW5kZXIgc3lzdGVtcyB3aGljaCBtb2RlbCB0aGUgcmVjb21tZW5kYXRpb24gcHJvY2VzcyBhcwphIHByZWRpY3Rpb24gcHJvYmxlbS4gV2UgY29uc2lkZXIgZGlmZmVyZW50IHR5cGVzIG9mIGJpYXNlcyB0aGF0IGNhbiBvY2N1cgppbiB0aGlzIHNldHRpbmcsIGRpc2N1c3MgdmFyaW91cyBmYWlybmVzcyBkZWZpbml0aW9ucywgYW5kIGFsc28gcHJvcG9zZSBhCm5vdmVsIGJpYXMgbWl0aWdhdGlvbiBzdHJhdGVneSB0byBhZGRyZXNzIHBvdGVudGlhbCB1bmZhaXJuZXNzIGluIGEKcmF0aW5nLWJhc2VkIHJlY29tbWVuZGVyIHN5c3RlbS4gQmFzZWQgb24gYW4gYW5hbHlzaXMgb2YgZmFpcm5lc3MgbWV0cmljcwp1c2VkIGluIG1hY2hpbmUgbGVhcm5pbmcgYW5kIGEgZGlzY3Vzc2lvbiBvZiB0aGVpciBhcHBsaWNhYmlsaXR5IGluIHRoZQpyZWNvbW1lbmRlciBzeXN0ZW0gZG9tYWluLCB3ZSBtYXAgdGhlIHByb3Bvc2VkIG1ldHJpY3MgZnJvbSB0aGUgdHdvCmRvbWFpbnMgYW5kIGlkZW50aWZ5IGNvbW1vbmx5IHVzZWQgY29uY2VwdHMgYW5kIGRlZmluaXRpb25zIG9mIGZhaXJuZXNzLgpGaW5hbGx5LCB0byBhZGRyZXNzIHVuZmFpcm5lc3MgYW5kIHBvdGVudGlhbCBiaWFzIGFnYWluc3QgY2VydGFpbiBncm91cHMKaW4gYSByZWNvbW1lbmRlciBzeXN0ZW0sIHdlIGRldmVsb3AgYSBiaWFzIG1pdGlnYXRpb24gYWxnb3JpdGhtIGFuZApjb25kdWN0IGNhc2Ugc3R1ZGllcyBvbiBvbmUgc3ludGhldGljIGFuZCBvbmUgZW1waXJpY2FsIGRhdGFzZXQgdG8gc2hvdwppdHMgZWZmZWN0aXZlbmVzcy4gT3VyIHJlc3VsdHMgc2hvdyB0aGF0IHVuZmFpcm5lc3MgY2FuIGJlIHNpZ25pZmljYW50bHkKbG93ZXJlZCB0aHJvdWdoIG91ciBhcHByb2FjaCBhbmQgdGhhdCBiaWFzIG1pdGlnYXRpb24gaXMgYSBmcnVpdGZ1bCBhcmVhCm9mIHJlc2VhcmNoIGZvciByZWNvbW1lbmRlciBzeXN0ZW1zLn0sCgludW1iZXIgPSB7NX0sCglqb3VybmFsID0ge0luZi4gUHJvY2Vzcy4gTWFuYWcufSwKCWF1dGhvciA9IHtBc2hva2FuLCBBc2h3YXRoeSBhbmQgSGFhcywgQ2hyaXN0aWFufSwKCW1vbnRoID0gc2VwLAoJeWVhciA9IHsyMDIxfSwKCWtleXdvcmRzID0ge0FsZ29yaXRobWljIGZhaXJuZXNzLCBCaWFzIG1pdGlnYXRpb24sIEZhaXJuZXNzIG1ldHJpY3MsIFJlY29tbWVuZGVyIHN5c3RlbXN9LAoJcGFnZXMgPSB7MTAyNjQ2fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tjaGVuX2ltcGFjdF8yMDIwLAoJdGl0bGUgPSB7SW1wYWN0IG9mIHtFbmR9LXtVc2VyfSB7UHJpdmFjeX0ge0VuaGFuY2luZ30ge1RlY2hub2xvZ2llc30gKHtQRVRzfSkgb24ge0Zpcm1zfeKAmSB7QW5hbHl0aWNzfSB7UGVyZm9ybWFuY2V9fSwKCXVybCA9IHtodHRwczovL2Fpc2VsLmFpc25ldC5vcmcvaWNpczIwMjAvZGlnaXRhbF9jb21tZXJjZS9kaWdpdGFsX2NvbW1lcmNlLzMvfSwKCWFic3RyYWN0ID0ge0JpZyBkYXRhIGFuYWx5dGljcyBpbiBkaWdpdGFsIGNvbW1lcmNlIHJlcXVpcmVzIHZhc3QgYW1vdW50cyBvZiBwZXJzb25hbAppbmZvcm1hdGlvbiBmcm9tIGNvbnN1bWVycywgYnV0IHRoaXMgZ2l2ZXMgcmlzZSB0byBtYWpvciBwcml2YWN5IGNvbmNlcm5zLgpUbyBjb21iYXQgdGhlIHRocmVhdCBvZiBwcml2YWN5IGludmFzaW9uLCBpbmRpdmlkdWFscyBhcmUgcHJvYWN0aXZlbHkKYWRvcHRpbmcgcHJpdmFjeSBlbmhhbmNpbmcgdGVjaG5vbG9naWVzIChQRVRzKSB0byBwcm90ZWN0IHRoZWlyIHBlcnNvbmFsCmluZm9ybWF0aW9uLiBDb25zdW1lcnPigJkgYWRvcHRpb24gb2YgUEVUcyBtYXkgaGFtcGVyIGZpcm1z4oCZIGJpZyBkYXRhCmFuYWx5dGljcyBjYXBhYmlsaXRpZXMgYW5kIHBlcmZvcm1hbmNlIGJ1dCBvdXIga25vd2xlZGdlIG9mIGhvdyBQRVRzCmltcGFjdCBmaXJtc+KAmSBkYXRhIGFuYWx5dGljcyBpcyBsaW1pdGVkLiBUaGlzIHN0dWR5IHByb3Bvc2VzIGFuCmluZHVjdGl2ZWx5IGRlcml2ZWQgZnJhbWV3b3JrIHdoaWNoIHF1YWxpdGF0aXZlbHkgc2hvd3MgdGhhdCBlbmQtdXNlciBQRVRzCmluZHVjZSBtZWFzdXJlbWVudCBlcnJvciBhbmQvb3IgbWlzc2luZyB2YWx1ZXMgd2l0aCByZWdhcmRzIHRvIGF0dHJpYnV0ZXMsCmVudGl0aWVzIGFuZCByZWxhdGlvbnNoaXBzIGluIGZpcm1z4oCZIGN1c3RvbWVyIGRhdGFiYXNlcywgYnV0IHRoZSBpbXBhY3RzCm9mIHNwZWNpZmljIGVuZC11c2VyIFBFVHMgbWF5IHZhcnkgYnkgYW5hbHl0aWNzIHVzZSBjYXNlLiBPdXIgc2ltdWxhdGlvbgpleHBlcmltZW50cyBpbiB0aGUgY29udGV4dCBvZiBwcm9kdWN0IHJlY29tbWVuZGF0aW9ucyBxdWFudGl0aXZlbHkgZmluZAp0aGF0IGNvbnN1bWVyc+KAmSBhZG9wdGlvbiBjaGFyYWN0ZXJpc3RpY3MgKGFkb3B0aW9uIHJhdGUgYW5kIHBhdHRlcm4pIGFuZApQRVRzIGNoYXJhY3RlcmlzdGljcyAocHJvdGVjdGlvbiBtZWNoYW5pc20gYW5kIGludGVuc2l0eSkgc2lnbmlmaWNhbnRseQphZmZlY3QgdGhlIHBlcmZvcm1hbmNlIG9mIHJlY29tbWVuZGVyIHN5c3RlbXMufSwKCXVybGRhdGUgPSB7MjAyMS0wMS0yOH0sCglhdXRob3IgPSB7Q2hlbiwgRGF3ZWkgYW5kIEhhaG4sIEp1bmdwaWx9LAoJeWVhciA9IHsyMDIwfSwKfQoKCgpAbWFzdGVyc3RoZXNpc3tuYXJheWFuX3doYXRfMjAxOSwKCXRpdGxlID0ge1doYXQgaXMgdGhlIHtWYWx1ZX0gb2Yge1JhdGluZ30ge09ic2N1cmV9IHtJdGVtc30/IHtBbn0ge0FuYWx5c2lzfSBvZiB0aGUge0VmZmVjdH0gb2Yge0xlc3N9LXtQb3B1bGFyfSB7SXRlbXN9IG9uIHtSZWNvbW1lbmRhdGlvbn0ge1F1YWxpdHl9fSwKCXVybCA9IHtodHRwczovL2NvbnNlcnZhbmN5LnVtbi5lZHUvaGFuZGxlLzExMjk5LzIwNjE0OH0sCglhYnN0cmFjdCA9IHtSZWNvbW1lbmRlciBzeXN0ZW1zIGRlc2lnbmVycyBiZWxpZXZlIHRoYXQgdGhlIHN5c3RlbSBzdGFuZHMgdG8gYmVuZWZpdApmcm9tIHRoZSB1c2VycyByYXRpbmcgaXRlbXMgdGhhdCBkbyBub3QgaGF2ZSBtYW55IHJhdGluZ3MuIEhvd2V2ZXIsIHRoZQplZmZlY3Qgb2YgdGhpcyBhY3Qgb2YgcmF0aW5nIGxlc3NlciBrbm93biBpdGVtcyBvbiB0aGUgdXNlcidzCnJlY29tbWVuZGF0aW9ucyBpcyB1bmtub3duLiBUaGlzIGxlYWRzIHRvIGFza2luZyB0aGUgcXVlc3Rpb24gb2Ygd2hldGhlcgp0aGVzZSBsb3cgcG9wdWxhcml0eSBpdGVtcyBhZmZlY3QgdGhlIHJlY29tbWVuZGF0aW9ucyByZWNlaXZlZCBieSB1c2Vycy4KVGhpcyB3b3JrIGxvb2tzIGF0IHRoZSBlZmZlY3QgbGVzcyBwb3B1bGFyIGl0ZW1zIGhhdmUgb24gYSB1c2VyJ3MKcmVjb21tZW5kYXRpb25zIGFuZCB0aGUgcHJlZGljdGlvbiBhbmQgcmVjb21tZW5kYXRpb25zIG1ldHJpY3MgdGhhdApxdWFudGlmeSB0aGUgcXVhbGl0eSBvZiByZWNvbW1lbmRhdGlvbnMg4oCmfSwKCXNjaG9vbCA9IHtVbml2ZXJzaXR5IG9mIE1pbm5lc290YX0sCglhdXRob3IgPSB7TmFyYXlhbiwgQX0sCgl5ZWFyID0gezIwMTl9LAoJbm90ZSA9IHtQdWJsaWNhdGlvbiBUaXRsZTogRGVwYXJ0bWVudCBvZiBDb21wdXRlciBTY2llbmNlClZvbHVtZTogTS5TLiBpbiBDb21wdXRlciBTY2llbmNlfSwKfQoKCgpAaW5wcm9jZWVkaW5nc3twZXNzZW1pZXJfdXNpbmdfMjAxOSwKCXRpdGxlID0ge1VzaW5nIGZhY2lhbCByZWNvZ25pdGlvbiBzZXJ2aWNlcyBhcyBpbXBsaWNpdCBmZWVkYmFjayBmb3IgcmVjb21tZW5kZXJzfSwKCXZvbHVtZSA9IHsyNDUwfSwKCXVybCA9IHtodHRwOi8vY2V1ci13cy5vcmcvVm9sLTI0NTAvcGFwZXI0LnBkZn0sCglhYnN0cmFjdCA9IHtVc2VyIGF1dGhlbnRpY2F0aW9uIGFuZCBmZWVkYmFjayBnYXRoZXJpbmcgYXJlIGNydWNpYWwgYXNwZWN0cyBmb3IKcmVjb21tZW5kZXIgc3lzdGVtcy4gVGhlIG1vc3QgY29tbW9uIGltcGxlbWVudGF0aW9ucywgYSB1c2VybmFtZSAvCnBhc3N3b3JkIGxvZ2luIGFuZCBzdGFyIHJhdGluZyBzeXN0ZW1zLCByZXF1aXJlIHVzZXIgaW50ZXJhY3Rpb24gYW5kIGEKY29nbml0aXZlIGVmZm9ydCBmcm9tIHRoZSB1c2VyLiBBcyBhIHJlc3VsdCwgdXNlcnMgb3B0IHRvIHNhdmUgdGhlaXIKcGFzc3dvcmQgaW4gdGhlIGludGVyZmFjZSBhbmQgb3B0aW9uYWwgZmVlZGJhY2sgd2l0aCBhIHN0YXIgcmF0aW5nIHN5c3RlbQppcyBvZnRlbiBza2lwcGVkLCBlc3BlY2lhbGx5IGZvciBhcHBsaWNhdGlvbnMgc3VjaCBhcyB2aWRlbyB3YXRjaGluZyBpbiBhCmhvbWUgZW52aXJvbm1lbnQuIEluIHRoaXMgYXJ0aWNsZSwgd2UgcHJvcG9zZSBhbiBhbHRlcm5hdGl2ZSBtZXRob2QgZm9yCnVzZXIgYXV0aGVudGljYXRpb24gYmFzZWQgb24gZmFjaWFsIHJlY29nbml0aW9uIGFuZCBhbiBhdXRvbWF0aWMgZmVlZGJhY2sKZ2F0aGVyaW5nIG1ldGhvZCBieSBkZXRlY3RpbmcgdmFyaW91cyBmYWNlIGNoYXJhY3RlcmlzdGljcy4gVXNpbmcgZmFjaWFsCnJlY29nbml0aW9uIHdpdGggYSBjYW1lcmEgaW4gYSB0YWJsZXQsIHNtYXJ0cGhvbmUsIG9yIHNtYXJ0IFRWLCB0aGUKcGVyc29ucyBpbiBmcm9udCBvZiB0aGUgc2NyZWVuIGNhbiBiZSBpZGVudGlmaWVkIGluIG9yZGVyIHRvIGxpbmsgdmlkZW8Kd2F0Y2hpbmcgc2Vzc2lvbnMgdG8gdGhlaXIgdXNlciBwcm9maWxlLiBEdXJpbmcgdmlkZW8gd2F0Y2hpbmcsIGltcGxpY2l0CmZlZWRiYWNrIGlzIGF1dG9tYXRpY2FsbHkgZ2F0aGVyZWQgdGhyb3VnaCBlbW90aW9uIHJlY29nbml0aW9uLCBhdHRlbnRpb24KbWVhc3VyZW1lbnRzLCBhbmQgYmVoYXZpb3IgYW5hbHlzaXMuIEFuIGVtb3Rpb24gZmluZ2VycHJpbnQsIHdoaWNoIGlzCmRlZmluZWQgYXMgYSB1bmlxdWUgc3BlY3RydW0gb2YgZXhwZWN0ZWQgZW1vdGlvbnMgZm9yIGEgdmlkZW8gc2NlbmUsIGlzCmNvbXBhcmVkIHRvIHRoZSByZWNvZ25pemVkIGVtb3Rpb25zIGluIG9yZGVyIHRvIGVzdGltYXRlIHRoZSBleHBlcmllbmNlIG9mCmEgdXNlciB3aGlsZSB3YXRjaGluZy4gQW4gZXZhbHVhdGlvbiB3aXRoIGEgdGVzdCBwYW5lbCBzaG93ZWQgdGhhdApoYXBwaW5lc3MgY2FuIGJlIG1vc3QgYWNjdXJhdGVseSBkZXRlY3RlZCBhbmQgdGhlIHJlY29nbml6ZWQgZW1vdGlvbnMgYXJlCmNvcnJlbGF0ZWQgd2l0aCB0aGUgdXNlcuKAmXMgc3RhciByYXRpbmcufSwKCXB1Ymxpc2hlciA9IHtDRVVSLVdTfSwKCWF1dGhvciA9IHtQZXNzZW1pZXIsIFRvb24gRGUgYW5kIENvcHBlbnMsIEluZSBhbmQgTWFydGVucywgTHVjfSwKCXllYXIgPSB7MjAxOX0sCglwYWdlcyA9IHs4fSwKfQoKCgpAaW5wcm9jZWVkaW5nc3tsdV9zdGFuZGluZ18yMDIxLAoJdGl0bGUgPSB7U3RhbmRpbmcgaW4ge1lvdXJ9IHtTaG9lc306IHtFeHRlcm5hbH0ge0Fzc2Vzc21lbnRzfSBmb3Ige1BlcnNvbmFsaXplZH0ge1JlY29tbWVuZGVyfSB7U3lzdGVtc319LAoJdXJsID0ge2h0dHA6Ly9keC5kb2kub3JnLzEwLjExNDUvMzQwNDgzNS4zNDYyOTE2fSwKCWRvaSA9IHsxMC4xMTQ1LzM0MDQ4MzUuMzQ2MjkxNn0sCglhdXRob3IgPSB7THUsIEhvbmd5dSBhbmQgTWEsIFdlaXpoaSBhbmQgWmhhbmcsIE1pbiBhbmQgZGUgUmlqa2UsIE1hYXJ0ZW4gYW5kIExpdSwgWWlxdW4gYW5kIE1hLCBTaGFvcGluZ30sCgl5ZWFyID0gezIwMjF9LAp9CgoKCkBpbnByb2NlZWRpbmdze3BhdWxsaWVyX3JlY29tbWVuZGVyXzIwMjAsCgl0aXRsZSA9IHtBIHtSZWNvbW1lbmRlcn0ge1N5c3RlbXN94oCZIGFsZ29yaXRobSBldmFsdWF0aW9uIHVzaW5nIHRoZSB7TGVuc2tpdH0gbGlicmFyeSBhbmQge01vdmllTGVuc30gZGF0YWJhc2VzfSwKCXVybCA9IHtodHRwOi8vZHguZG9pLm9yZy8xMC4xMTA5L0JNU0I0OTQ4MC4yMDIwLjkzNzk5MTR9LAoJZG9pID0gezEwLjExMDkvQk1TQjQ5NDgwLjIwMjAuOTM3OTkxNH0sCglhYnN0cmFjdCA9IHtBIHdpZGUgdmFyaWV0eSBvZiBhbGdvcml0aG1zIGNhbiBiZSBmb3VuZCBpbiB0aGUgcmVjb21tZW5kZXIgc3lzdGVtJ3MKbGl0ZXJhdHVyZSBmb3IgY29tcHV0aW5nIHByZWRpY3Rpb25zIGFuZCBvYnRhaW5pbmcgcmVjb21tZW5kYXRpb24gbGlzdHMuClRoZXNlIGFsZ29yaXRobXMnIHBlcmZvcm1hbmNlIGNhbiBiZSBldmFsdWF0ZWQgYnkgbnVtZXJvdXMgbWV0cmljcwpkZXBlbmRpbmcgb24gd2hpY2ggYnVzaW5lc3MgZ29hbCBpcyBpbnRlbmRlZCB0byBiZSBvcHRpbWl6ZWQuIEhvd2V2ZXIsCnBlcmZvcm1hbmNlIGNhbiB2YXJ5IGRyYXN0aWNhbGx5IGRlcGVuZGluZyBvbiB0aGUgc2VsZWN0ZWQgYWxnb3JpdGhtIGFzCndlbGwgYXMgdGhlIHNlbGVjdGVkIG1ldHJpYy4gSW4gYWRkaXRpb24sIGFsZ29yaXRobWljIHBlcmZvcm1hbmNlIGNhbiBiZQpoZWF2aWx5IGluZmx1ZW5jZWQgYnkgdGhlIGV4cGVyaW1lbnRhbCBzZXR1cCBzdWNoIGFzOiBkYXRhc2V0LApjcm9zcy12YWxpZGF0aW9uIHN0cmF0ZWd5LCByZWNvbW1lbmRlciBmcmFtZXdvcmsgYW5kIHRoZSBtb2RlbApoeXBlci1wYXJhbWV0ZXJzIGNob3Nlbi4gSW4gdGhpcyB3b3JrIHdlIHdpbGwgY29tcGFyZSBuaW5lIGRpZmZlcmVudAphbGdvcml0aG1zIHVzZWQgZm9yIGdlbmVyYXRpbmcgcmVjb21tZW5kYXRpb25zIGFjcm9zcyBzZXZlbiBkaWZmZXJlbnQKbWV0cmljcy4gT25lIHZpdGFsIGFzcGVjdCBvZiBvdXIgYW5hbHlzaXMgaXMgdG8gcHJvdmlkZSBhIGRldGFpbGVkIGFuZApjbGVhciBleHBsYW5hdGlvbiBvZiB0aGUgbWV0aG9kb2xvZ3kgY2FycmllZCBvdXQgYXMgd2VsbCBhcyB0aGUKZXhwZXJpbWVudGFsIHNldHVwIGluIG9yZGVyIHRvIGVuc3VyZSBmdXR1cmUgcmVwcm9kdWNpYmlsaXR5LiBNb3Jlb3ZlciwKcGVyZm9ybWFuY2UgYW5kIGNvbXB1dGF0aW9uIHRpbWVzIHdpbGwgYmUgaW50cm9kdWNlZCBmb3IgZnVydGhlcgpjb21wYXJpc29uLiBGaW5hbGx5LCBvcHRpbWFsIG1vZGVsIGh5cGVyLXBhcmFtZXRlciBjb21iaW5hdGlvbnMgd2lsbCBiZQpwcmVzZW50ZWQgZm9yIGJlbmNobWFya2luZyBwdXJwb3Nlcy59LAoJYXV0aG9yID0ge1BhdWxsaWVyLCBBbGVqbyBhbmQgU290ZWxvLCBSYWZhZWx9LAoJbW9udGggPSBvY3QsCgl5ZWFyID0gezIwMjB9LAoJa2V5d29yZHMgPSB7QWxnb3JpdGhtcywgQnVzaW5lc3MsIENvbXB1dGF0aW9uYWwgbW9kZWxpbmcsIERhdGFiYXNlcywgRXZhbHVhdGlvbiwgTGVuc0tpdCwgTGlicmFyaWVzLCBNb3ZpZUxlbnMsIE11bHRpbWVkaWEgY29tbXVuaWNhdGlvbiwgUHJlZGljdGlvbiBhbGdvcml0aG1zLCBSZWNvbW1lbmRlciBzeXN0ZW1zLCBSZXByb2R1Y2liaWxpdHkgb2YgcmVzdWx0c30sCglwYWdlcyA9IHsxLS03fSwKfQoKCgpAbWFzdGVyc3RoZXNpc3tub2d1ZXJhX3RvcnJlc19zaXN0ZW1hXzIwMjEsCgl0aXRsZSA9IHtTaXN0ZW1hIGRlIHJlY29tZW5kYWNpw7NuIGRlIG1hdHLDrWN1bGEgZGUgY3Vyc29zIGVsZWN0aXZvcyBwYXJhIGVzdHVkaWFudGVzIGRlIHtJbmdlbmllcsOtYX0ge0VsZWN0csOzbmljYX0gZSB7SW5nZW5pZXLDrWF9IGRlIHtUZWxlY29tdW5pY2FjaW9uZXN9IGRlIGxhIHtVTkFEfX0sCgl1cmwgPSB7aHR0cHM6Ly9yZXBvc2l0b3J5LnVuYWQuZWR1LmNvL2hhbmRsZS8xMDU5Ni80MDQ2NX0sCgl1cmxkYXRlID0gezIwMjEtMDUtMTJ9LAoJc2Nob29sID0ge1VuaXZlcnNpZGFkIE5hY2lvbmFsIEFiaWVydGEgeSBhIERpc3RhbmNpYX0sCglhdXRob3IgPSB7Tm9ndWVyYSBUb3JyZXMsIEFkcmlhbmEgZGVsIFBpbGFyfSwKCWNvbGxhYm9yYXRvciA9IHtSw7phIFDDqXJleiwgU2FudGlhZ299LAoJbW9udGggPSBhcHIsCgl5ZWFyID0gezIwMjF9LAoJbm90ZSA9IHtWb2x1bWU6IE1TIGluIEluZm9ybWF0aW9uIFRlY2hub2xvZ3l9LAp9CgoKCkBpbmNvbGxlY3Rpb257emllZ2xlcl9lbXBmZWhsdW5nc3N5c3RlbWVfMjAyMCwKCWFkZHJlc3MgPSB7V2llc2JhZGVufSwKCXRpdGxlID0ge0VtcGZlaGx1bmdzc3lzdGVtZX0sCglpc2JuID0gezk3OC0zLTY1OC0xNzI5MS02fSwKCXVybCA9IHtodHRwczovL2RvaS5vcmcvMTAuMTAwNy85NzgtMy02NTgtMTcyOTEtNl81Mn0sCglhYnN0cmFjdCA9IHtFbXBmZWhsdW5nc3N5c3RlbWUgc3RlbGxlbiBoZXV0ZSBlaW5lIHplbnRyYWxlIEtvbXBvbmVudGUgdmllbGVyCk9ubGluZS1QbGF0dGZvcm1lbiBkYXIsIGRpZSBiZWkgT25saW5lLVNob3BzIHVuZCB2aWVsZW4gYW5kZXJlbgpBbndlbmR1bmdlbiBow6R1ZmlnIHp1bSBFaW5zYXR6IGtvbW10LiBaaWVsIGlzdCBlcywgZGVtIEt1bmRlbiBlbnRzcHJlY2hlbmQKc2VpbmVuIHBlcnPDtm5saWNoZW4gUHLDpGZlcmVuemVuIFByb2R1a3RlIG9kZXIgYW5kZXJlIEFydGlrZWwKdm9yenVzY2hsYWdlbiwgZGllIGbDvHIgaWhuIHZvbiBJbnRlcmVzc2Ugc2luZCB1bmQgcG90ZW56aWVsbCB6dSBlaW5lbSBLYXVmCm9kZXIgZ2VuZXJlbGwgenVyIE51dHp1bmcgZsO8aHJlbi4gRW1wZmVobHVuZ3NzeXN0ZW1lIGhhYmVuIGVpbmUgZXJoZWJsaWNoZQp3aXJ0c2NoYWZ0bGljaGUgQmVkZXV0dW5nLCBkYSBzaWUgaW4gdmllbGVuIEbDpGxsZW4genUgZWluZW0gc2lnbmlmaWthbnRlbgpBbnRlaWwgenUgRXJmb2xnc2Zha3RvcmVuIHdpZSBDbGljay10aHJvdWdoLVJhdGVuIG9kZXIgS8OkdWZlbiBiZWl0cmFnZW4uCldpciBzdGVsbGVuIGluIGRpZXNlbSBLYXBpdGVsIGRpZSB1bnRlcnNjaGllZGxpY2hlbiBBbnPDpHR6ZSB6dXIKYXV0b21hdGlzaWVydGVuIEVtcGZlaGx1bmdzZ2VidW5nIHZvciB1bmQgYmVzY2hyZWliZW4ga29ua3JldGUgVGVjaG5pa2VuCnp1IGRlcmVuIFVtc2V0enVuZy4gV2VpdGVyaGluIGdlaGVuIHdpciBhdWYgd2VzZW50bGljaGUgQXNwZWt0ZSBkZXIKR2VzdGFsdHVuZyB1bmQgQmV3ZXJ0dW5nIHZvbiBFbXBmZWhsdW5nc3N5c3RlbWVuIGVpbiB1bmQgZGlza3V0aWVyZW4KYW53ZW5kZXJyZWxldmFudGUgVGhlbWVuIHdpZSBVc2FiaWxpdHkgdW5kIFZlcnRyYXVlbiBpbiBzeXN0ZW1nZW5lcmllcnRlCkVtcGZlaGx1bmdlbi59LAoJYm9va3RpdGxlID0ge0hhbmRidWNoIHtEaWdpdGFsZX0ge1dpcnRzY2hhZnR9fSwKCXB1Ymxpc2hlciA9IHtTcHJpbmdlciBGYWNobWVkaWVuIFdpZXNiYWRlbn0sCglhdXRob3IgPSB7WmllZ2xlciwgSsO8cmdlbiBhbmQgTG9lcHAsIEJlbmVkaWt0fSwKCWVkaXRvciA9IHtLb2xsbWFubiwgVG9iaWFzfSwKCXllYXIgPSB7MjAyMH0sCglkb2kgPSB7MTAuMTAwNy85NzgtMy02NTgtMTcyOTEtNl81Mn0sCglwYWdlcyA9IHs3MTctLTc0MX0sCn0KCgoKQGlucHJvY2VlZGluZ3N7d2FuZ19iYXllc2lhbl8yMDE5LAoJdGl0bGUgPSB7QmF5ZXNpYW4ge0RlZXB9IHtMZWFybmluZ30ge0Jhc2VkfSB7RXhwbG9yYXRpb259LXtFeHBsb2l0YXRpb259IGZvciB7UGVyc29uYWxpemVkfSB7UmVjb21tZW5kYXRpb25zfX0sCgl1cmwgPSB7aHR0cDovL2R4LmRvaS5vcmcvMTAuMTEwOS9JQ1RBSS4yMDE5LjAwMjUzfSwKCWRvaSA9IHsxMC4xMTA5L0lDVEFJLjIwMTkuMDAyNTN9LAoJYWJzdHJhY3QgPSB7UGVyc29uYWxpemVkIFJlY29tbWVuZGF0aW9uIFN5c3RlbXMgcmVxdWlyZSBhbiBlZmZlY3RpdmUgbWV0aG9kIHRvIGJhbGFuY2UKZXhwbG9yYXRpb24gYW5kIGV4cGxvaXRhdGlvbi4gVG8gbGVhcm4gZWZmZWN0aXZlIHN0cmF0ZWdpZXMsIHVzZXIgYW5kIGl0ZW0KYXR0cmlidXRlcyBhcmUgY3JpdGljYWwgZGF0YSBzb3VyY2VzIHRvIGNhcHR1cmUgY29udGV4dHVhbCBpbmZvcm1hdGlvbi4gSW4KdGhpcyBwYXBlciwgd2UgZmlyc3QgcHJlc2VudCBhbiBhcHByb2FjaCBiYXNlZCBvbiBCYXllc2lhbiBEZWVwIExlYXJuaW5nCnRvIGxlYXJuIGEgY29tcGFjdCByZXByZXNlbnRhdGlvbiBvZiB1c2VyIGFuZCBpdGVtIGF0dHJpYnV0ZXMgdG8gZ3VpZGUKZXhwbG9pdGF0aW9uLiBBIGtleSBub3ZlbHR5IG9mIHRoZSBhcHByb2FjaCBsaWVzIGluIGl0cyBhYmlsaXR5IHRvIGFsc28KY2FwdHVyZSB0aGUgdW5jZXJ0YWludHkgYXNzb2NpYXRlZCB3aXRoIHRoZSBtb2RlbCBvdXRwdXQgdG8gZ3VpZGUKZXhwbG9yYXRpb24uIFdlIHRoZW4gc2hvdyBob3cgdG8gZnVydGhlciBib29zdCBleHBsb3JhdGlvbiBieQppbmNvcnBvcmF0aW5nIG1vZGVsIHVuY2VydGFpbnR5IHdpdGggdGhhdCBvZiBkYXRhIHVuY2VydGFpbnR5LgpFeHBlcmltZW50YWwgcmVzdWx0cyBkZW1vbnN0cmF0ZSB0aGUgYmVuZWZpdHMgb2Ygb3VyIGFwcHJvYWNoIGluIHRlcm1zIG9mCmFjY3VyYWN5IGluIHJlY29tbWVuZGF0aW9ucyBhcyB3ZWxsIGFzIGl0cyBwZXJmb3JtYW5jZSBpbiBhbiBvbmxpbmUKc2V0dGluZy59LAoJcHVibGlzaGVyID0ge2llZWV4cGxvcmUuaWVlZS5vcmd9LAoJYXV0aG9yID0ge1dhbmcsIFggYW5kIEthZGlvZ2x1LCBTfSwKCW1vbnRoID0gbm92LAoJeWVhciA9IHsyMDE5fSwKCWtleXdvcmRzID0ge0JheWVzaWFuIERlZXAgTGVhcm5pbmcsIEV4cGxvcmF0aW9uRXhwbG9pdGF0aW9uLCBQZXJzb25hbGl6ZWQgUmVjb21tZW5kYXRpb24sIEJheWVzaWFuIGRlZXAgbGVhcm5pbmcsIGJvb3N0IGV4cGxvcmF0aW9uLCBjb21wYWN0IHJlcHJlc2VudGF0aW9uLCBjb250ZXh0dWFsIGluZm9ybWF0aW9uLCBjcml0aWNhbCBkYXRhIHNvdXJjZXMsIGRhdGEgdW5jZXJ0YWludHksIGV4cGxvcmF0aW9uLWV4cGxvaXRhdGlvbiwgaW5mb3JtYXRpb24gZmlsdGVyaW5nLCBpdGVtIGF0dHJpYnV0ZXMsIGxlYXJuaW5nIChhcnRpZmljaWFsIGludGVsbGlnZW5jZSksIG1vZGVsIG91dHB1dCwgbW9kZWwgdW5jZXJ0YWludHksIHBlcnNvbmFsaXplZCByZWNvbW1lbmRhdGlvbiBzeXN0ZW1zLCByZWNvbW1lbmRlciBzeXN0ZW1zfSwKCXBhZ2VzID0gezE3MTUtLTE3MTl9LAp9CgoKCkBhcnRpY2xle2RlX3Blc3NlbWllcl9ldmFsdWF0aW5nXzIwMjAsCgl0aXRsZSA9IHtFdmFsdWF0aW5nIGZhY2lhbCByZWNvZ25pdGlvbiBzZXJ2aWNlcyBhcyBpbnRlcmFjdGlvbiB0ZWNobmlxdWUgZm9yIHJlY29tbWVuZGVyIHN5c3RlbXN9LAoJaXNzbiA9IHsxMzgwLTc1MDF9LAoJdXJsID0ge2h0dHBzOi8vZG9pLm9yZy8xMC4xMDA3L3MxMTA0Mi0wMjAtMDkwNjEtOH0sCglkb2kgPSB7MTAuMTAwNy9zMTEwNDItMDIwLTA5MDYxLTh9LAoJYWJzdHJhY3QgPSB7UmVjb21tZW5kZXIgc3lzdGVtcyBhcmUgdG9vbHMgYW5kIHRlY2huaXF1ZXMgdG8gYXNzaXN0IHVzZXJzIGluIHRoZQpjb250ZW50IHNlbGVjdGlvbiBwcm9jZXNzIHRoZXJlYnkgY29waW5nIHdpdGggdGhlIHByb2JsZW0gb2YgaW5mb3JtYXRpb24Kb3ZlcmxvYWQuIEZvciByZWNvbW1lbmRlciBzeXN0ZW1zLCB1c2VyIGF1dGhlbnRpY2F0aW9uIGFuZCBmZWVkYmFjawpnYXRoZXJpbmcgYXJlIG9mIGNydWNpYWwgaW1wb3J0YW5jZS4gSG93ZXZlciwgdGhlIHR5cGljYWwgdXNlcgphdXRoZW50aWNhdGlvbiB3aXRoIHVzZXJuYW1lIC8gcGFzc3dvcmQgYW5kIGZlZWRiYWNrIG1ldGhvZCB3aXRoIGEgc3RhcgpyYXRpbmcgc3lzdGVtIGFyZSBub3QgdXNlciBmcmllbmRseSBhbmQgb2Z0ZW4gYnlwYXNzZWQuIFRoaXMgYXJ0aWNsZQpwcm9wb3NlcyBhbiBhbHRlcm5hdGl2ZSBtZXRob2QgZm9yIHVzZXIgYXV0aGVudGljYXRpb24gYmFzZWQgb24gZmFjaWFsCnJlY29nbml0aW9uIGFuZCBhbiBhdXRvbWF0aWMgZmVlZGJhY2sgZ2F0aGVyaW5nIG1ldGhvZCBieSBkZXRlY3RpbmcKdmFyaW91cyBmYWNlIGNoYXJhY3RlcmlzdGljcyBzdWNoIGFzIGVtb3Rpb25zLiBXZSBzdHVkaWVkIHRoZSB1c2UgY2FzZSBvZgp2aWRlbyB3YXRjaGluZy4gUGhvdG9zIG1hZGUgd2l0aCB0aGUgZnJvbnQtZmFjaW5nIGNhbWVyYSBvZiBhIHRhYmxldCwKc21hcnRwaG9uZSwgb3Igc21hcnQgVFYgYXJlIHVzZWQgYXMgaW5wdXQgb2YgYSBmYWNpYWwgcmVjb2duaXRpb24gc2VydmljZS4KVGhlIHBlcnNvbnMgaW4gZnJvbnQgb2YgdGhlIHNjcmVlbiBjYW4gYmUgaWRlbnRpZmllZC4gRHVyaW5nIHZpZGVvCndhdGNoaW5nLCBpbXBsaWNpdCBmZWVkYmFjayBmb3IgdGhlIHZpZGVvIGNvbnRlbnQgaXMgYXV0b21hdGljYWxseQpnYXRoZXJlZCB0aHJvdWdoIGVtb3Rpb24gcmVjb2duaXRpb24sIGF0dGVudGlvbiBtZWFzdXJlbWVudHMsIGFuZCBiZWhhdmlvcgphbmFseXNpcy4gQW4gZXZhbHVhdGlvbiB3aXRoIGEgdGVzdCBwYW5lbCBzaG93ZWQgdGhhdCB0aGUgcmVjb2duaXplZAplbW90aW9ucyBhcmUgY29ycmVsYXRlZCB3aXRoIHRoZSB1c2Vy4oCZcyBzdGFyIHJhdGluZ3MgYW5kIHRoYXQgaGFwcGluZXNzCmNhbiBiZSBtb3N0IGFjY3VyYXRlbHkgZGV0ZWN0ZWQuIFNvIGFzIHRoZSBtYWluIGNvbnRyaWJ1dGlvbiwgdGhpcyBhcnRpY2xlCmluZGljYXRlcyB0aGF0IGVtb3Rpb24gcmVjb2duaXRpb24gbWlnaHQgYmUgdXNlZCBhcyBhbiBhbHRlcm5hdGl2ZQpmZWVkYmFjayBtZWNoYW5pc20gZm9yIHJlY29tbWVuZGVyIHN5c3RlbXMufSwKCWpvdXJuYWwgPSB7TXVsdGltZWQuIFRvb2xzIEFwcGwufSwKCWF1dGhvciA9IHtEZSBQZXNzZW1pZXIsIFRvb24gYW5kIENvcHBlbnMsIEluZSBhbmQgTWFydGVucywgTHVjfSwKCW1vbnRoID0ganVuLAoJeWVhciA9IHsyMDIwfSwKfQoKCgpAaW5jb2xsZWN0aW9ue3ZhcmdhX3JlY29tbWVuZGVyXzIwMTksCgl0aXRsZSA9IHtSZWNvbW1lbmRlciB7U3lzdGVtc319LAoJaXNibiA9IHs5NzgtMS00ODQyLTQ4NTgtNH0sCgl1cmwgPSB7aHR0cDovL2R4LmRvaS5vcmcvMTAuMTAwNy85NzgtMS00ODQyLTQ4NTktMX0sCglib29rdGl0bGUgPSB7UHJhY3RpY2FsIHtEYXRhfSB7U2NpZW5jZX0gd2l0aCB7UHl0aG9ufSAzfSwKCXB1Ymxpc2hlciA9IHtBcHJlc3N9LAoJYXV0aG9yID0ge1ZhcmdhLCBFcnZpbn0sCgl5ZWFyID0gezIwMTl9LAoJZG9pID0gezEwLjEwMDcvOTc4LTEtNDg0Mi00ODU5LTF9LAoJcGFnZXMgPSB7MzE3LS0zMzl9LAp9CgoKCkBtYXN0ZXJzdGhlc2lze2JhbHNlY2FfbmluZXpfZXN0dWRpb18yMDE5LAoJdGl0bGUgPSB7RXN0dWRpbyBkZWwgZXN0YWRvIGRlbCBhcnRlIGRlIGxhIGNpZW5jaWEgZGUgZGF0b3MgYXBsaWNhZGEgYSBsYSBuZXVyb2Vjb25vbcOtYX0sCgl1cmwgPSB7aHR0cDovL2RzcGFjZS51cHMuZWR1LmVjL2hhbmRsZS8xMjM0NTY3ODkvMTc2MDV9LAoJc2Nob29sID0ge1VuaXZlcnNpZGFkIFBvbGl0w6ljbmljYSBTYWxlc2lhbmF9LAoJYXV0aG9yID0ge0JhbHNlY2EgTsOtw7FleiwgQ2hyaXN0aWFuIFJpY2FyZG8gYW5kIEZlcm7DoW5kZXogUGXDsWFmaWVsLCBEYW5pZWwgQW5kcsOpc30sCgl5ZWFyID0gezIwMTl9LAp9CgoKCkBhcnRpY2xle25nX2NicmVjXzIwMjAsCgl0aXRsZSA9IHt7Q0JSZWN9OiBhIGJvb2sgcmVjb21tZW5kYXRpb24gc3lzdGVtIGZvciBjaGlsZHJlbiB1c2luZyB0aGUgbWF0cml4IGZhY3RvcmlzYXRpb24gYW5kIGNvbnRlbnQtYmFzZWQgZmlsdGVyaW5nIGFwcHJvYWNoZXN9LAoJdm9sdW1lID0gezE2fSwKCWlzc24gPSB7MTc0My04MTg3fSwKCXVybCA9IHtodHRwczovL3d3dy5pbmRlcnNjaWVuY2VvbmxpbmUuY29tL2RvaS9hYnMvMTAuMTUwNC9JSkJJRE0uMjAyMC4xMDQ3Mzh9LAoJZG9pID0gezEwLjE1MDQvSUpCSURNLjIwMjAuMTA0NzM4fSwKCWFic3RyYWN0ID0ge1Byb21vdGluZyBnb29kIHJlYWRpbmcgaGFiaXRzIGFtb25nIGNoaWxkcmVuIGlzIGVzc2VudGlhbCwgZ2l2ZW4gdGhlCmVub3Jtb3VzIGluZmx1ZW5jZSBvZiByZWFkaW5nIG9uIHN0dWRlbnRzJyBkZXZlbG9wbWVudCBhcyBsZWFybmVycyBhbmQKbWVtYmVycyBvZiB0aGUgc29jaWV0eS4gVW5mb3J0dW5hdGVseSwgdmVyeSBmZXcgKGNoaWxkcmVuKSB3ZWJzaXRlcyBvcgpvbmxpbmUgYXBwbGljYXRpb25zIHJlY29tbWVuZCBib29rcyB0byBjaGlsZHJlbiwgZXZlbiB0aG91Z2ggdGhleSBjYW4gcGxheQphIHNpZ25pZmljYW50IHJvbGUgaW4gZW5jb3VyYWdpbmcgY2hpbGRyZW4gdG8gcmVhZC4gR2l2ZW4gdGhhdCBhIGZldwpwb3B1bGFyIGJvb2sgd2Vic2l0ZXMgc3VnZ2VzdCBib29rcyB0byBjaGlsZHJlbiBiYXNlZCBvbiB0aGUgcG9wdWxhcml0eSBvZgpib29rcyBvciByYW5raW5ncyBvbiBib29rcywgdGhleSBhcmUgbm90IGN1c3RvbWlzZWQvcGVyc29uYWxpc2VkIGZvciBlYWNoCmluZGl2aWR1YWwgdXNlciBhbmQgbGlrZWx5IHJlY29tbWVuZCBib29rcyB0aGF0IHVzZXJzIGRvIG5vdCB3YW50IG9yIGxpa2UuCldlIGhhdmUgaW50ZWdyYXRlZCB0aGUgbWF0cml4IGZhY3RvcmlzYXRpb24gYXBwcm9hY2ggYW5kIHRoZSBjb250ZW50LWJhc2VkCmFwcHJvYWNoLCBpbiBhZGRpdGlvbiB0byBwcmVkaWN0aW5nIHRoZSBncmFkZSBsZXZlbHMgb2YgYm9va3MsIHRvCnJlY29tbWVuZCBib29rcyBmb3IgY2hpbGRyZW4uIFJlY2VudCByZXNlYXJjaCB3b3JrcyBoYXZlIGRlbW9uc3RyYXRlZCB0aGF0CmEgaHlicmlkIGFwcHJvYWNoLCB3aGljaCBjb21iaW5lcyBkaWZmZXJlbnQgZmlsdGVyaW5nIGFwcHJvYWNoZXMsIGlzIG1vcmUKZWZmZWN0aXZlIGluIG1ha2luZyByZWNvbW1lbmRhdGlvbnMuIENvbmR1Y3RlZCBlbXBpcmljYWwgc3R1ZHkgaGFzCnZlcmlmaWVkIHRoZSBlZmZlY3RpdmVuZXNzIG9mIG91ciBwcm9wb3NlZCBjaGlsZHJlbiBib29rIHJlY29tbWVuZGF0aW9uCnN5c3RlbS59LAoJbnVtYmVyID0gezJ9LAoJam91cm5hbCA9IHtJbnRlcm5hdGlvbmFsIEpvdXJuYWwgb2YgQnVzaW5lc3MgSW50ZWxsaWdlbmNlIGFuZCBEYXRhIE1pbmluZ30sCglhdXRob3IgPSB7TmcsIFlpdS1LYWl9LAoJbW9udGggPSBqYW4sCgl5ZWFyID0gezIwMjB9LAoJbm90ZSA9IHtQdWJsaXNoZXI6IEluZGVyc2NpZW5jZSBQdWJsaXNoZXJzfSwKCXBhZ2VzID0gezEyOS0tMTQ5fSwKfQo=\">\n            <i class=\"fa fa-download fa-fw\"></i> Download BibTeX\n          </a>\n        </li>\n        <li>\n          <a href=\"//bibbase.org/rss?bib=https://api.zotero.org/users/6655/collections/3TB3KT36/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&amp;format=bibtex&amp;limit=100\">\n            <i class=\"fa fa-rss fa-fw\"></i> RSS Feed\n          </a>\n          </li>\n      </ul>\n      </li>\n\n      \n\n\n      \n    </ul>\n\n\n    <div class=\"alert alert-success bibbase_msg\" id=\"bibbase_msg_embed\"\n         style=\"display: none;\">\n\n      Excellent! Next you can\n      <a href=\"/network/register?next=/network/newSiteFromTemplate%3Fbiburl=https://api.zotero.org/users/6655/collections/3TB3KT36/items?key=VFvZhZXIoHNBbzoLZ1IM2zgf&amp;format=bibtex&amp;limit=100\"\n      >create a new website with this list</a>, or\n      embed it in an existing web page by copying &amp; pasting\n      any of the following snippets.\n\n      <div style=\"text-align: left; margin-top: 1em;\">\n        <strong>JavaScript</strong>\n        <span class=\"comment recommended\">(easiest)</span>\n        <div class=\"well well-small\">\n          <code>\n            &lt;script src=\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F3TB3KT36%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1&amp;jsonp=1\"&gt;&lt;/script&gt;\n          </code>\n        </div>\n\n        <strong>PHP</strong>\n        <div class=\"well well-small\">\n          <code>\n            &lt;?php\n            $contents = file_get_contents(\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F3TB3KT36%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1\");\n            print_r($contents);\n            ?&gt;\n          </code>\n        </div>\n\n        <strong>iFrame</strong>\n        <span class=\"comment\">(not recommended)</span>\n        <div class=\"well well-small\">\n          <code>\n            &lt;iframe src=\"https://bibbase.org/show?bib=https%3A%2F%2Fapi.zotero.org%2Fusers%2F6655%2Fcollections%2F3TB3KT36%2Fitems%3Fkey%3DVFvZhZXIoHNBbzoLZ1IM2zgf%26format%3Dbibtex%26limit%3D100&amp;jsonp=1\"&gt;&lt;/iframe&gt;\n          </code>\n        </div>\n\n        <p>\n          For more details see the <a href=\"//bibbase.org/help\">documention</a>.\n        </p>\n      </div>\n    </div>\n\n    <div class=\"alert alert-success bibbase_msg\" id=\"bibbase_msg_preview\"\n         style=\"display: none;\">\n\n      This is a preview! To use this list on your own web site\n      or create a new web site from it,\n      <a href=\"/network/register?next=/network/newAccountWithFile%3FfileId=\"\n      >create a free account</a>. The file will be added\n      and you will be able to edit it in the File Manager.\n      We will show you instructions once you've created your account.\n    </div>\n\n    <div class=\"alert alert-warning bibbase_msg\" id=\"bibbase_msg_mendeley1\"\n         style=\"display: none;\">\n\n      <p>To the site owner:</p>\n\n      <p><strong>Action required!</strong> Mendeley is changing its\n        API. In order to keep using Mendeley with BibBase past April\n        14th, you need to:\n        <ol>\n          <li>renew the authorization for BibBase on Mendeley, and</li>\n          <li>update the BibBase URL\n            in your page the same way you did when you initially set up\n            this page.\n          </li>\n        </ol>\n      </p>\n\n      <p>\n        <a href=\"http://bibbase.org/cgi-bin/mendeley2/get.py\">\n          <i class=\"fa fa-angle-double-right\"></i>\n          Fix it now</a>\n      </p>\n    </div>\n\n  </div>\n\n\n  <div id=\"bibbase_body\">\n    \n  \n  <div class=\"bibbase_group_whole\" id=\"group_2024\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2024')\"\n      onkeypress=\"toggleGroup('group_2024')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2024</span>\n      <span class=\"bibbase_group_count\">\n        \n        (5)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'https://md.ekstrandom.net/pubs/ecir-fair-grids', event);\">\n    Towards optimizing ranking in grid-layout for provider-side fairness.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  In <i>Advances in Information Retrieval</i>, volume 14612, of <i>LNCS</i>, pages 90–105, March 2024. Springer\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-fair-grids\"\n     onclick=\"log_download('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'https://md.ekstrandom.net/pubs/ecir-fair-grids', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Towards optimizing ranking in grid-layout for provider-side fairness [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-031-56069-9_7\"\n     onclick=\"log_download('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024', 'http://doi.org/10.1007/978-3-031-56069-9_7', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-ekstrand-towardsoptimizingrankingingridlayoutforprovidersidefairness-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_towards_2024,\n\tseries = {{LNCS}},\n\ttitle = {Towards optimizing ranking in grid-layout for provider-side fairness},\n\tvolume = {14612},\n\tcopyright = {All rights reserved},\n\turl = {https://md.ekstrandom.net/pubs/ecir-fair-grids},\n\tdoi = {10.1007/978-3-031-56069-9_7},\n\tabstract = {Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.},\n\tlanguage = {en},\n\turldate = {2024-01-04},\n\tbooktitle = {Advances in {Information} {Retrieval}},\n\tpublisher = {Springer},\n\tauthor = {Raj, Amifa and Ekstrand, Michael D.},\n\tmonth = mar,\n\tyear = {2024},\n\tpages = {90--105},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Information access systems, such as search engines and recommender systems, order and position results based on their estimated relevance. These results are then evaluated for a range of concerns, including provider-side fairness: whether exposure to users is fairly distributed among items and the people who created them. Several fairness-aware ranking and re-ranking techniques have been proposed to ensure fair exposure for providers, but this work focuses almost exclusively on linear layouts in which items are displayed in single ranked list. Many widely-used systems use other layouts, such as the grid views common in streaming platforms, image search, and other applications. Providing fair exposure to providers in such layouts is not well-studied. We seek to fill this gap by providing a grid-aware re-ranking algorithm to optimize layouts for provider-side fairness by adapting existing re-ranking techniques to grid-aware browsing models, and an analysis of the effect of grid-specific factors such as device size on the resulting fairness optimization.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-mcp\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024', 'https://md.ekstrandom.net/pubs/ecir-mcp', event);\">\n    Multiple testing for IR and recommendation system experiments.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ihemelandu, N.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 46th European Conference on Information Retrieval</i>, volume 14610, of <i>LNCS</i>, pages 449–457, March 2024. Springer\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/ecir-mcp\"\n     onclick=\"log_download('ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024', 'https://md.ekstrandom.net/pubs/ecir-mcp', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Multiple testing for IR and recommendation system experiments [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-031-56063-7_37\"\n     onclick=\"log_download('ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024', 'http://doi.org/10.1007/978-3-031-56063-7_37', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ihemelandu-ekstrand-multipletestingforirandrecommendationsystemexperiments-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{ihemelandu_multiple_2024,\n\tseries = {{LNCS}},\n\ttitle = {Multiple testing for {IR} and recommendation system experiments},\n\tvolume = {14610},\n\turl = {https://md.ekstrandom.net/pubs/ecir-mcp},\n\tdoi = {10.1007/978-3-031-56063-7_37},\n\tabstract = {While there has been significant research on statistical techniques for comparing two information retrieval (IR) systems, many IR experiments test more than two systems. This can lead to inflated false discoveries due to the multiple-comparison problem (MCP). A few IR studies have investigated multiple comparison procedures; these studies mostly use TREC data and control the familywise error rate. In this study, we extend their investigation to include recommendation system evaluation data as well as multiple comparison procedures that controls for False Discovery Rate (FDR).},\n\tlanguage = {en},\n\turldate = {2024-01-04},\n\tbooktitle = {Proceedings of the 46th {European} {Conference} on {Information} {Retrieval}},\n\tpublisher = {Springer},\n\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\n\tmonth = mar,\n\tyear = {2024},\n\tpages = {449--457},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  While there has been significant research on statistical techniques for comparing two information retrieval (IR) systems, many IR experiments test more than two systems. This can lead to inflated false discoveries due to the multiple-comparison problem (MCP). A few IR studies have investigated multiple comparison procedures; these studies mostly use TREC data and control the familywise error rate. In this study, we extend their investigation to include recommendation system evaluation data as well as multiple comparison procedures that controls for False Discovery Rate (FDR).\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wegmeth-vente-purucker-revealingthehiddenimpactoftopnmetricsonoptimizationinrecommendersystems-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  Revealing the Hidden Impact of Top-N Metrics on Optimization in Recommender Systems.\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wegmeth, L.; Vente, T.;  and Purucker, L.\n</span>\n\n  \n\n</span>\n\n  In Goharian, N.; Tonellotto, N.; He, Y.; Lipani, A.; McDonald, G.; Macdonald, C.;  and Ounis, I., editor(s), <i>Advances in Information Retrieval</i>, pages 140–156,  2024. Springer Nature Switzerland\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-031-56027-9_9\"\n     onclick=\"log_download('wegmeth-vente-purucker-revealingthehiddenimpactoftopnmetricsonoptimizationinrecommendersystems-2024', 'http://doi.org/10.1007/978-3-031-56027-9_9', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wegmeth-vente-purucker-revealingthehiddenimpactoftopnmetricsonoptimizationinrecommendersystems-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wegmeth-vente-purucker-revealingthehiddenimpactoftopnmetricsonoptimizationinrecommendersystems-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{wegmeth_revealing_2024,\n\ttitle = {Revealing the {Hidden} {Impact} of {Top}-{N} {Metrics} on {Optimization} in {Recommender} {Systems}},\n\tisbn = {978-3-031-56027-9},\n\tdoi = {10.1007/978-3-031-56027-9_9},\n\tabstract = {The hyperparameters of recommender systems for top-n predictions are typically optimized to enhance the predictive performance of algorithms. Thereby, the optimization algorithm, e.g., grid search or random search, searches for the best hyperparameter configuration according to an optimization-target metric, like nDCG or Precision. In contrast, the optimized algorithm, e.g., Alternating Least Squares Matrix Factorization or Bayesian Personalized Ranking, internally optimizes a different loss function during training, like squared error or cross-entropy. To tackle this discrepancy, recent work focused on generating loss functions better suited for recommender systems. Yet, when evaluating an algorithm using a top-n metric during optimization, another discrepancy between the optimization-target metric and the training loss has so far been ignored. During optimization, the top-n items are selected for computing a top-n metric; ignoring that the top-n items are selected from the recommendations of a model trained with an entirely different loss function. Item recommendations suitable for optimization-target metrics could be outside the top-n recommended items; hiddenly impacting the optimization performance. Therefore, we were motivated to analyze whether the top-n items are optimal for optimization-target top-n metrics. In pursuit of an answer, we exhaustively evaluate the predictive performance of 250 selection strategies besides selecting the top-n. We extensively evaluate each selection strategy over twelve implicit feedback and eight explicit feedback data sets with eleven recommender systems algorithms. Our results show that there exist selection strategies other than top-n that increase predictive performance for various algorithms and recommendation domains. However, the performance of the top \\$\\${\\textbackslash}sim 43{\\textbackslash}\\%\\$\\$∼43\\%of selection strategies is not significantly different. We discuss the impact of our findings on optimization and re-ranking in recommender systems and feasible solutions. The implementation of our study is publicly available.},\n\tlanguage = {en},\n\tbooktitle = {Advances in {Information} {Retrieval}},\n\tpublisher = {Springer Nature Switzerland},\n\tauthor = {Wegmeth, Lukas and Vente, Tobias and Purucker, Lennart},\n\teditor = {Goharian, Nazli and Tonellotto, Nicola and He, Yulan and Lipani, Aldo and McDonald, Graham and Macdonald, Craig and Ounis, Iadh},\n\tyear = {2024},\n\tpages = {140--156},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  The hyperparameters of recommender systems for top-n predictions are typically optimized to enhance the predictive performance of algorithms. Thereby, the optimization algorithm, e.g., grid search or random search, searches for the best hyperparameter configuration according to an optimization-target metric, like nDCG or Precision. In contrast, the optimized algorithm, e.g., Alternating Least Squares Matrix Factorization or Bayesian Personalized Ranking, internally optimizes a different loss function during training, like squared error or cross-entropy. To tackle this discrepancy, recent work focused on generating loss functions better suited for recommender systems. Yet, when evaluating an algorithm using a top-n metric during optimization, another discrepancy between the optimization-target metric and the training loss has so far been ignored. During optimization, the top-n items are selected for computing a top-n metric; ignoring that the top-n items are selected from the recommendations of a model trained with an entirely different loss function. Item recommendations suitable for optimization-target metrics could be outside the top-n recommended items; hiddenly impacting the optimization performance. Therefore, we were motivated to analyze whether the top-n items are optimal for optimization-target top-n metrics. In pursuit of an answer, we exhaustively evaluate the predictive performance of 250 selection strategies besides selecting the top-n. We extensively evaluate each selection strategy over twelve implicit feedback and eight explicit feedback data sets with eleven recommender systems algorithms. Our results show that there exist selection strategies other than top-n that increase predictive performance for various algorithms and recommendation domains. However, the performance of the top $$\\sim 43\\%$$∼43%of selection strategies is not significantly different. We discuss the impact of our findings on optimization and re-ranking in recommender systems and feasible solutions. The implementation of our study is publicly available.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"pathak-spezzano-anempiricalanalysisofinterventionstrategieseffectivenessforcounteringmisinformationamplificationbyrecommendationalgorithms-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  An Empirical Analysis of Intervention Strategies’ Effectiveness for Countering Misinformation Amplification by Recommendation Algorithms.\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Pathak, R.;  and Spezzano, F.\n</span>\n\n  \n\n</span>\n\n  In Goharian, N.; Tonellotto, N.; He, Y.; Lipani, A.; McDonald, G.; Macdonald, C.;  and Ounis, I., editor(s), <i>Advances in Information Retrieval</i>, volume 14611, of <i>LNCS</i>, pages 285–301,  2024. Springer Nature Switzerland\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-031-56066-8_23\"\n     onclick=\"log_download('pathak-spezzano-anempiricalanalysisofinterventionstrategieseffectivenessforcounteringmisinformationamplificationbyrecommendationalgorithms-2024', 'http://doi.org/10.1007/978-3-031-56066-8_23', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/pathak-spezzano-anempiricalanalysisofinterventionstrategieseffectivenessforcounteringmisinformationamplificationbyrecommendationalgorithms-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('pathak-spezzano-anempiricalanalysisofinterventionstrategieseffectivenessforcounteringmisinformationamplificationbyrecommendationalgorithms-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{pathak_empirical_2024,\n\tseries = {{LNCS}},\n\ttitle = {An {Empirical} {Analysis} of {Intervention} {Strategies}’ {Effectiveness} for {Countering} {Misinformation} {Amplification} by {Recommendation} {Algorithms}},\n\tvolume = {14611},\n\tisbn = {978-3-031-56066-8},\n\tdoi = {10.1007/978-3-031-56066-8_23},\n\tabstract = {Social network platforms connect people worldwide, facilitating communication, information sharing, and personal/professional networking. They use recommendation algorithms to personalize content and enhance user experiences. However, these algorithms can unintentionally amplify misinformation by prioritizing engagement over accuracy. For instance, recent works suggest that popularity-based and network-based recommendation algorithms contribute the most to misinformation diffusion. In our study, we present an exploration on two Twitter datasets to understand the impact of intervention techniques on combating misinformation amplification initiated by recommendation algorithms. We simulate various scenarios and evaluate the effectiveness of intervention strategies in social sciences such as Virality Circuit Breakers and accuracy nudges. Our findings highlight that these intervention strategies are generally successful when applied on top of collaborative filtering and content-based recommendation algorithms, while having different levels of effectiveness depending on the number of users keen to spread fake news present in the dataset.},\n\tlanguage = {en},\n\tbooktitle = {Advances in {Information} {Retrieval}},\n\tpublisher = {Springer Nature Switzerland},\n\tauthor = {Pathak, Royal and Spezzano, Francesca},\n\teditor = {Goharian, Nazli and Tonellotto, Nicola and He, Yulan and Lipani, Aldo and McDonald, Graham and Macdonald, Craig and Ounis, Iadh},\n\tyear = {2024},\n\tpages = {285--301},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Social network platforms connect people worldwide, facilitating communication, information sharing, and personal/professional networking. They use recommendation algorithms to personalize content and enhance user experiences. However, these algorithms can unintentionally amplify misinformation by prioritizing engagement over accuracy. For instance, recent works suggest that popularity-based and network-based recommendation algorithms contribute the most to misinformation diffusion. In our study, we present an exploration on two Twitter datasets to understand the impact of intervention techniques on combating misinformation amplification initiated by recommendation algorithms. We simulate various scenarios and evaluate the effectiveness of intervention strategies in social sciences such as Virality Circuit Breakers and accuracy nudges. Our findings highlight that these intervention strategies are generally successful when applied on top of collaborative filtering and content-based recommendation algorithms, while having different levels of effectiveness depending on the number of users keen to spread fake news present in the dataset.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0957417423016664\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024', 'https://www.sciencedirect.com/science/article/pii/S0957417423016664', event);\">\n    Recommendations with minimum exposure guarantees: a post-processing framework.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Lopes, R.; Alves, R.; Ledent, A.; Santos, R. L. T.;  and Kloft, M.\n</span>\n\n  \n\n</span>\n\n  <i>Expert Systems with Applications</i>, 236: 121164. February 2024.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0957417423016664\"\n     onclick=\"log_download('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024', 'https://www.sciencedirect.com/science/article/pii/S0957417423016664', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Recommendations with minimum exposure guarantees: a post-processing framework [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1016/j.eswa.2023.121164\"\n     onclick=\"log_download('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024', 'http://doi.org/10.1016/j.eswa.2023.121164', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2024')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{lopes_recommendations_2024,\n\ttitle = {Recommendations with minimum exposure guarantees: a post-processing framework},\n\tvolume = {236},\n\tissn = {0957-4174},\n\tshorttitle = {Recommendations with minimum exposure guarantees},\n\turl = {https://www.sciencedirect.com/science/article/pii/S0957417423016664},\n\tdoi = {10.1016/j.eswa.2023.121164},\n\tabstract = {Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantaged groups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.},\n\turldate = {2023-09-19},\n\tjournal = {Expert Systems with Applications},\n\tauthor = {Lopes, Ramon and Alves, Rodrigo and Ledent, Antoine and Santos, Rodrygo L. T. and Kloft, Marius},\n\tmonth = feb,\n\tyear = {2024},\n\tkeywords = {Exposure, Fairness, Integer linear programming, Recommender systems, to-read},\n\tpages = {121164},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantaged groups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2023\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2023')\"\n      onkeypress=\"toggleGroup('group_2023')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2023</span>\n      <span class=\"bibbase_group_count\">\n        \n        (14)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2309.11723\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023', 'http://arxiv.org/abs/2309.11723', event);\">\n    Candidate set sampling for evaluating top-N recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ihemelandu, N.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 22nd IEEE/WIC international conference on web intelligence and intelligent agent technology</i>, pages 88–94, October 2023. \n  <span class=\"note\">arXiv:2309.11723 [cs]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2309.11723\"\n     onclick=\"log_download('ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023', 'http://arxiv.org/abs/2309.11723', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Candidate set sampling for evaluating top-N recommendation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1109/WI-IAT59888.2023.00018\"\n     onclick=\"log_download('ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023', 'http://doi.org/10.1109/WI-IAT59888.2023.00018', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ihemelandu-ekstrand-candidatesetsamplingforevaluatingtopnrecommendation-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{ihemelandu_candidate_2023,\n\ttitle = {Candidate set sampling for evaluating top-{N} recommendation},\n\turl = {http://arxiv.org/abs/2309.11723},\n\tdoi = {10.1109/WI-IAT59888.2023.00018},\n\tabstract = {The strategy for selecting candidate sets -- the set of items that the recommendation system is expected to rank for each user -- is an important decision in carrying out an offline top-\\$N\\$ recommender system evaluation. The set of candidates is composed of the union of the user&#x27;s test items and an arbitrary number of non-relevant items that we refer to as decoys. Previous studies have aimed to understand the effect of different candidate set sizes and selection strategies on evaluation. In this paper, we extend this knowledge by studying the specific interaction of candidate set selection strategies with popularity bias, and use simulation to assess whether sampled candidate sets result in metric estimates that are less biased with respect to the true metric values under complete data that is typically unavailable in ordinary experiments.},\n\turldate = {2023-11-08},\n\tbooktitle = {Proceedings of the 22nd {IEEE}/{WIC} international conference on web intelligence and intelligent agent technology},\n\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\n\tmonth = oct,\n\tyear = {2023},\n\tnote = {arXiv:2309.11723 [cs]},\n\tkeywords = {Computer Science - Information Retrieval},\n\tpages = {88--94},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  The strategy for selecting candidate sets – the set of items that the recommendation system is expected to rank for each user – is an important decision in carrying out an offline top-$N$ recommender system evaluation. The set of candidates is composed of the union of the user's test items and an arbitrary number of non-relevant items that we refer to as decoys. Previous studies have aimed to understand the effect of different candidate set sizes and selection strategies on evaluation. In this paper, we extend this knowledge by studying the specific interaction of candidate set selection strategies with popularity bias, and use simulation to assess whether sampled candidate sets result in metric estimates that are less biased with respect to the true metric values under complete data that is typically unavailable in ordinary experiments.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1007/s41060-020-00241-1\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023', 'https://doi.org/10.1007/s41060-020-00241-1', event);\">\n    Modeling uncertainty to improve personalized recommendations via Bayesian deep learning.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wang, X.;  and Kadıoğlu, S.\n</span>\n\n  \n\n</span>\n\n  <i>International Journal of Data Science and Analytics</i>, 16(2): 191–201. August 2023.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1007/s41060-020-00241-1\"\n     onclick=\"log_download('wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023', 'https://doi.org/10.1007/s41060-020-00241-1', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Modeling uncertainty to improve personalized recommendations via Bayesian deep learning [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s41060-020-00241-1\"\n     onclick=\"log_download('wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023', 'http://doi.org/10.1007/s41060-020-00241-1', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wang-kadolu-modelinguncertaintytoimprovepersonalizedrecommendationsviabayesiandeeplearning-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{wang_modeling_2023,\n\ttitle = {Modeling uncertainty to improve personalized recommendations via {Bayesian} deep learning},\n\tvolume = {16},\n\tissn = {2364-4168},\n\turl = {https://doi.org/10.1007/s41060-020-00241-1},\n\tdoi = {10.1007/s41060-020-00241-1},\n\tabstract = {Modeling uncertainty has been a major challenge in developing Machine Learning solutions to solve real world problems in various domains. In Recommender Systems, a typical usage of uncertainty is to balance exploration and exploitation, where the uncertainty helps to guide the selection of new options in exploration. Recent advances in combining Bayesian methods with deep learning enable us to express uncertain status in deep learning models. In this paper, we investigate an approach based on Bayesian deep learning to improve personalized recommendations. We first build deep learning architectures to learn useful representation of user and item inputs for predicting their interactions. We then explore multiple embedding components to accommodate different types of user and item inputs. Based on Bayesian deep learning techniques, a key novelty of our approach is to capture the uncertainty associated with the model output and further utilize it to boost exploration in the context of Recommender Systems. We test the proposed approach in both a Collaborative Filtering and a simulated online recommendation setting. Experimental results on publicly available benchmarks demonstrate the benefits of our approach in improving the recommendation performance.},\n\tlanguage = {en},\n\tnumber = {2},\n\turldate = {2024-03-17},\n\tjournal = {International Journal of Data Science and Analytics},\n\tauthor = {Wang, Xin and Kadıoğlu, Serdar},\n\tmonth = aug,\n\tyear = {2023},\n\tpages = {191--201},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Modeling uncertainty has been a major challenge in developing Machine Learning solutions to solve real world problems in various domains. In Recommender Systems, a typical usage of uncertainty is to balance exploration and exploitation, where the uncertainty helps to guide the selection of new options in exploration. Recent advances in combining Bayesian methods with deep learning enable us to express uncertain status in deep learning models. In this paper, we investigate an approach based on Bayesian deep learning to improve personalized recommendations. We first build deep learning architectures to learn useful representation of user and item inputs for predicting their interactions. We then explore multiple embedding components to accommodate different types of user and item inputs. Based on Bayesian deep learning techniques, a key novelty of our approach is to capture the uncertainty associated with the model output and further utilize it to boost exploration in the context of Recommender Systems. We test the proposed approach in both a Collaborative Filtering and a simulated online recommendation setting. Experimental results on publicly available benchmarks demonstrate the benefits of our approach in improving the recommendation performance.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wegmeth-vente-purucker-beel-theeffectofrandomseedsfordatasplittingonrecommendationaccuracy-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  The effect of random seeds for data splitting on recommendation accuracy.\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wegmeth, L.; Vente, T.; Purucker, L.;  and Beel, J.\n</span>\n\n  \n\n</span>\n\n  In <i>Perspectives on the Evaluation of Recommender Systems Workshop (PERSPECTIVES 2023)</i>, September 2023. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wegmeth-vente-purucker-beel-theeffectofrandomseedsfordatasplittingonrecommendationaccuracy-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wegmeth-vente-purucker-beel-theeffectofrandomseedsfordatasplittingonrecommendationaccuracy-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{wegmeth_effect_2023,\n\ttitle = {The effect of random seeds for data splitting on recommendation accuracy},\n\tabstract = {The evaluation of recommender system algorithms depends on randomness, e.g., during randomly splitting data into training and testing data. We suspect that failing to account for randomness in this scenario may lead to misrepresenting the predictive accuracy of recommendation algorithms. To understand the community’s view of the importance of randomness, we conducted a paper study on 39 full papers published at the ACM RecSys 2022 conference. We found that the authors of 26 papers used some variation of a holdout split that requires a random seed. However, only five papers explicitly repeated experiments and averaged their results over different random seeds. This potentially problematic research practice motivated us to analyze the effect of data split random seeds on recommendation accuracy. Therefore, we train three common algorithms on nine public data sets with 20 data split random seeds, evaluate them on two ranking metrics with three different ranking cutoff values 𝑘, and compare the results. In the extreme case with 𝑘 = 1, we show that depending on the data split random seed, the accuracy with traditional recommendation algorithms deviates by up to ∼6.3\\% from the mean accuracy achieved on the data set. Hence, we show that an algorithm may significantly over- or under-perform when maliciously or negligently selecting a random seed for splitting the data. To showcase a mitigation strategy and better research practice, we compare holdout to cross-validation and show that, again, for 𝑘 = 1, the accuracy of algorithms evaluated with cross-validation deviates only up to ∼2.3\\% from the mean accuracy achieved on the data set. Furthermore, we found that the deviation becomes smaller the higher the value of 𝑘 for both holdout and cross-validation.},\n\tlanguage = {en},\n\tbooktitle = {Perspectives on the {Evaluation} of {Recommender} {Systems} {Workshop} ({PERSPECTIVES} 2023)},\n\tauthor = {Wegmeth, Lukas and Vente, Tobias and Purucker, Lennart and Beel, Joeran},\n\tmonth = sep,\n\tyear = {2023},\n\tkeywords = {to-read},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  The evaluation of recommender system algorithms depends on randomness, e.g., during randomly splitting data into training and testing data. We suspect that failing to account for randomness in this scenario may lead to misrepresenting the predictive accuracy of recommendation algorithms. To understand the community’s view of the importance of randomness, we conducted a paper study on 39 full papers published at the ACM RecSys 2022 conference. We found that the authors of 26 papers used some variation of a holdout split that requires a random seed. However, only five papers explicitly repeated experiments and averaged their results over different random seeds. This potentially problematic research practice motivated us to analyze the effect of data split random seeds on recommendation accuracy. Therefore, we train three common algorithms on nine public data sets with 20 data split random seeds, evaluate them on two ranking metrics with three different ranking cutoff values 𝑘, and compare the results. In the extreme case with 𝑘 = 1, we show that depending on the data split random seed, the accuracy with traditional recommendation algorithms deviates by up to ∼6.3% from the mean accuracy achieved on the data set. Hence, we show that an algorithm may significantly over- or under-perform when maliciously or negligently selecting a random seed for splitting the data. To showcase a mitigation strategy and better research practice, we compare holdout to cross-validation and show that, again, for 𝑘 = 1, the accuracy of algorithms evaluated with cross-validation deviates only up to ∼2.3% from the mean accuracy achieved on the data set. Furthermore, we found that the deviation becomes smaller the higher the value of 𝑘 for both holdout and cross-validation.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3604915.3610656\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023', 'https://dl.acm.org/doi/10.1145/3604915.3610656', event);\">\n    Introducing LensKit-Auto, an experimental automated recommender system (AutoRecSys) toolkit.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Vente, T.; Ekstrand, M.;  and Beel, J.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 17th ACM Conference on Recommender Systems</i>, of <i>RecSys '23</i>, pages 1212–1216, New York, NY, USA, September 2023. Association for Computing Machinery\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3604915.3610656\"\n     onclick=\"log_download('vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023', 'https://dl.acm.org/doi/10.1145/3604915.3610656', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Introducing LensKit-Auto, an experimental automated recommender system (AutoRecSys) toolkit [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3604915.3610656\"\n     onclick=\"log_download('vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023', 'http://doi.org/10.1145/3604915.3610656', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('vente-ekstrand-beel-introducinglenskitautoanexperimentalautomatedrecommendersystemautorecsystoolkit-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{vente_introducing_2023,\n\taddress = {New York, NY, USA},\n\tseries = {{RecSys} &#x27;23},\n\ttitle = {Introducing {LensKit}-{Auto}, an experimental automated recommender system ({AutoRecSys}) toolkit},\n\tisbn = {9798400702419},\n\turl = {https://dl.acm.org/doi/10.1145/3604915.3610656},\n\tdoi = {10.1145/3604915.3610656},\n\tabstract = {LensKit is one of the first and most popular Recommender System libraries. While LensKit offers a wide variety of features, it does not include any optimization strategies or guidelines on how to select and tune LensKit algorithms. LensKit developers have to manually include third-party libraries into their experimental setup or implement optimization strategies by hand to optimize hyperparameters. We found that 63.6\\% (21 out of 33) of papers using LensKit algorithms for their experiments did not select algorithms or tune hyperparameters. Non-optimized models represent poor baselines and produce less meaningful research results. This demo introduces LensKit-Auto. LensKit-Auto automates the entire Recommender System pipeline and enables LensKit developers to automatically select, optimize, and ensemble LensKit algorithms.},\n\turldate = {2023-09-18},\n\tbooktitle = {Proceedings of the 17th {ACM} {Conference} on {Recommender} {Systems}},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Vente, Tobias and Ekstrand, Michael and Beel, Joeran},\n\tmonth = sep,\n\tyear = {2023},\n\tkeywords = {Algorithm Selection, AutoRecSys, Automated Recommender Systems, CASH, Hyperparameter Optimization, Recommender Systems},\n\tpages = {1212--1216},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  LensKit is one of the first and most popular Recommender System libraries. While LensKit offers a wide variety of features, it does not include any optimization strategies or guidelines on how to select and tune LensKit algorithms. LensKit developers have to manually include third-party libraries into their experimental setup or implement optimization strategies by hand to optimize hyperparameters. We found that 63.6% (21 out of 33) of papers using LensKit algorithms for their experiments did not select algorithms or tune hyperparameters. Non-optimized models represent poor baselines and produce less meaningful research results. This demo introduces LensKit-Auto. LensKit-Auto automates the entire Recommender System pipeline and enables LensKit developers to automatically select, optimize, and ensemble LensKit algorithms.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2307.13632\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023', 'http://arxiv.org/abs/2307.13632', event);\">\n    Mitigating mainstream bias in recommendation via cost-sensitive learning.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Li, R. Z.; Urbano, J.;  and Hanjalic, A.\n</span>\n\n  \n\n</span>\n\n  July 2023.\n  <span class=\"note\">arXiv:2307.13632 [cs]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2307.13632\"\n     onclick=\"log_download('li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023', 'http://arxiv.org/abs/2307.13632', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Mitigating mainstream bias in recommendation via cost-sensitive learning [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3578337.3605134\"\n     onclick=\"log_download('li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023', 'http://doi.org/10.1145/3578337.3605134', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('li-urbano-hanjalic-mitigatingmainstreambiasinrecommendationviacostsensitivelearning-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@misc{li_mitigating_2023,\n\ttitle = {Mitigating mainstream bias in recommendation via cost-sensitive learning},\n\turl = {http://arxiv.org/abs/2307.13632},\n\tdoi = {10.1145/3578337.3605134},\n\tabstract = {Mainstream bias, where some users receive poor recommendations because their preferences are uncommon or simply because they are less active, is an important aspect to consider regarding fairness in recommender systems. Existing methods to mitigate mainstream bias do not explicitly model the importance of these non-mainstream users or, when they do, it is in a way that is not necessarily compatible with the data and recommendation model at hand. In contrast, we use the recommendation utility as a more generic and implicit proxy to quantify mainstreamness, and propose a simple user-weighting approach to incorporate it into the training process while taking the cost of potential recommendation errors into account. We provide extensive experimental results showing that quantifying mainstreamness via utility is better able at identifying non-mainstream users, and that they are indeed better served when training the model in a cost-sensitive way. This is achieved with negligible or no loss in overall recommendation accuracy, meaning that the models learn a better balance across users. In addition, we show that research of this kind, which evaluates recommendation quality at the individual user level, may not be reliable if not using enough interactions when assessing model performance.},\n\turldate = {2023-07-29},\n\tauthor = {Li, Roger Zhe and Urbano, Julián and Hanjalic, Alan},\n\tmonth = jul,\n\tyear = {2023},\n\tnote = {arXiv:2307.13632 [cs]},\n\tkeywords = {Computer Science - Information Retrieval},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Mainstream bias, where some users receive poor recommendations because their preferences are uncommon or simply because they are less active, is an important aspect to consider regarding fairness in recommender systems. Existing methods to mitigate mainstream bias do not explicitly model the importance of these non-mainstream users or, when they do, it is in a way that is not necessarily compatible with the data and recommendation model at hand. In contrast, we use the recommendation utility as a more generic and implicit proxy to quantify mainstreamness, and propose a simple user-weighting approach to incorporate it into the training process while taking the cost of potential recommendation errors into account. We provide extensive experimental results showing that quantifying mainstreamness via utility is better able at identifying non-mainstream users, and that they are indeed better served when training the model in a cost-sensitive way. This is achieved with negligible or no loss in overall recommendation accuracy, meaning that the models learn a better balance across users. In addition, we show that research of this kind, which evaluates recommendation quality at the individual user level, may not be reliable if not using enough interactions when assessing model performance.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3539618.3592004\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023', 'https://dl.acm.org/doi/10.1145/3539618.3592004', event);\">\n    Inference at scale: significance testing for large search and recommendation experiments.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ihemelandu, N.;  and Ekstrand, M. D.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval</i>, of <i>SIGIR '23</i>, pages 2087–2091, New York, NY, USA, July 2023. Association for Computing Machinery\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3539618.3592004\"\n     onclick=\"log_download('ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023', 'https://dl.acm.org/doi/10.1145/3539618.3592004', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Inference at scale: significance testing for large search and recommendation experiments [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3539618.3592004\"\n     onclick=\"log_download('ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023', 'http://doi.org/10.1145/3539618.3592004', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ihemelandu-ekstrand-inferenceatscalesignificancetestingforlargesearchandrecommendationexperiments-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{ihemelandu_inference_2023,\n\taddress = {New York, NY, USA},\n\tseries = {{SIGIR} &#x27;23},\n\ttitle = {Inference at scale: significance testing for large search and recommendation experiments},\n\tcopyright = {All rights reserved},\n\tisbn = {978-1-4503-9408-6},\n\tshorttitle = {Inference at scale},\n\turl = {https://dl.acm.org/doi/10.1145/3539618.3592004},\n\tdoi = {10.1145/3539618.3592004},\n\tabstract = {A number of information retrieval studies have been done to assess which statistical techniques are appropriate for comparing systems. However, these studies are focused on TREC-style experiments, which typically have fewer than 100 topics. There is no similar line of work for large search and recommendation experiments; such studies typically have thousands of topics or users and much sparser relevance judgements, so it is not clear if recommendations for analyzing traditional TREC experiments apply to these settings. In this paper, we empirically study the behavior of significance tests with large search and recommendation evaluation data. Our results show that the Wilcoxon and Sign tests show significantly higher Type-1 error rates for large sample sizes than the bootstrap, randomization and t-tests, which were more consistent with the expected error rate. While the statistical tests displayed differences in their power for smaller sample sizes, they showed no difference in their power for large sample sizes. We recommend the sign and Wilcoxon tests should not be used to analyze large scale evaluation results. Our result demonstrate that with Top-N recommendation and large search evaluation data, most tests would have a 100\\% chance of finding statistically significant results. Therefore, the effect size should be used to determine practical or scientific significance.},\n\turldate = {2023-07-23},\n\tbooktitle = {Proceedings of the 46th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Ihemelandu, Ngozi and Ekstrand, Michael D.},\n\tmonth = jul,\n\tyear = {2023},\n\tkeywords = {evaluation, statistical inference},\n\tpages = {2087--2091},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  A number of information retrieval studies have been done to assess which statistical techniques are appropriate for comparing systems. However, these studies are focused on TREC-style experiments, which typically have fewer than 100 topics. There is no similar line of work for large search and recommendation experiments; such studies typically have thousands of topics or users and much sparser relevance judgements, so it is not clear if recommendations for analyzing traditional TREC experiments apply to these settings. In this paper, we empirically study the behavior of significance tests with large search and recommendation evaluation data. Our results show that the Wilcoxon and Sign tests show significantly higher Type-1 error rates for large sample sizes than the bootstrap, randomization and t-tests, which were more consistent with the expected error rate. While the statistical tests displayed differences in their power for smaller sample sizes, they showed no difference in their power for large sample sizes. We recommend the sign and Wilcoxon tests should not be used to analyze large scale evaluation results. Our result demonstrate that with Top-N recommendation and large search evaluation data, most tests would have a 100% chance of finding statistically significant results. Therefore, the effect size should be used to determine practical or scientific significance.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3613455\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023', 'https://dl.acm.org/doi/10.1145/3613455', event);\">\n    Distributionally-informed recommender system evaluation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ekstrand, M. D.; Carterette, B.;  and Diaz, F.\n</span>\n\n  \n\n</span>\n\n  <i>ACM Transactions on Recommender Systems</i>. August 2023.\n  <span class=\"note\">Just Accepted</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3613455\"\n     onclick=\"log_download('ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023', 'https://dl.acm.org/doi/10.1145/3613455', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Distributionally-informed recommender system evaluation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3613455\"\n     onclick=\"log_download('ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023', 'http://doi.org/10.1145/3613455', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ekstrand-carterette-diaz-distributionallyinformedrecommendersystemevaluation-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{ekstrand_distributionally-informed_2023,\n\ttitle = {Distributionally-informed recommender system evaluation},\n\tcopyright = {All rights reserved},\n\turl = {https://dl.acm.org/doi/10.1145/3613455},\n\tdoi = {10.1145/3613455},\n\tabstract = {Current practice for evaluating recommender systems typically focuses on point estimates of user-oriented effectiveness metrics or business metrics, sometimes combined with additional metrics for considerations such as diversity and novelty. In this paper, we argue for the need for researchers and practitioners to attend more closely to various distributions that arise from a recommender system (or other information access system) and the sources of uncertainty that lead to these distributions. One immediate implication of our argument is that both researchers and practitioners must report and examine more thoroughly the distribution of utility between and within different stakeholder groups. However, distributions of various forms arise in many more aspects of the recommender systems experimental process, and distributional thinking has substantial ramifications for how we design, evaluate, and present recommender systems evaluation and research results. Leveraging and emphasizing distributions in the evaluation of recommender systems is a necessary step to ensure that the systems provide appropriate and equitably-distributed benefit to the people they affect.},\n\turldate = {2023-09-07},\n\tjournal = {ACM Transactions on Recommender Systems},\n\tauthor = {Ekstrand, Michael D. and Carterette, Ben and Diaz, Fernando},\n\tmonth = aug,\n\tyear = {2023},\n\tnote = {Just Accepted},\n\tkeywords = {distributions, evaluation, exposure, statistics},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Current practice for evaluating recommender systems typically focuses on point estimates of user-oriented effectiveness metrics or business metrics, sometimes combined with additional metrics for considerations such as diversity and novelty. In this paper, we argue for the need for researchers and practitioners to attend more closely to various distributions that arise from a recommender system (or other information access system) and the sources of uncertainty that lead to these distributions. One immediate implication of our argument is that both researchers and practitioners must report and examine more thoroughly the distribution of utility between and within different stakeholder groups. However, distributions of various forms arise in many more aspects of the recommender systems experimental process, and distributional thinking has substantial ramifications for how we design, evaluate, and present recommender systems evaluation and research results. Leveraging and emphasizing distributions in the evaluation of recommender systems is a necessary step to ensure that the systems provide appropriate and equitably-distributed benefit to the people they affect.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3591109\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023', 'https://doi.org/10.1145/3591109', event);\">\n    A Framework and Toolkit for Testing the Correctness of Recommendation Algorithms.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Michiels, L.; Verachtert, R.; Ferraro, A.; Falk, K.;  and Goethals, B.\n</span>\n\n  \n\n</span>\n\n  <i>ACM Trans. Recomm. Syst.</i>. April 2023.\n  <span class=\"note\">Place: New York, NY, USA Publisher: Association for Computing Machinery</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3591109\"\n     onclick=\"log_download('michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023', 'https://doi.org/10.1145/3591109', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"A Framework and Toolkit for Testing the Correctness of Recommendation Algorithms [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3591109\"\n     onclick=\"log_download('michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023', 'http://doi.org/10.1145/3591109', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>9 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('michiels-verachtert-ferraro-falk-goethals-aframeworkandtoolkitfortestingthecorrectnessofrecommendationalgorithms-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{michiels_framework_2023,\n\ttitle = {A {Framework} and {Toolkit} for {Testing} the {Correctness} of {Recommendation} {Algorithms}},\n\turl = {https://doi.org/10.1145/3591109},\n\tdoi = {10.1145/3591109},\n\tabstract = {Evaluating recommender systems adequately and thoroughly is an important\ntask. Significant efforts are dedicated to proposing metrics, methods and\nprotocols for doing so. However, there has been little discussion in the\nrecommender systems’ literature on the topic of testing. In this work, we\nadopt and adapt concepts from the software testing domain, e.g., code\ncoverage, metamorphic testing, or property-based testing, to help\nresearchers to detect and correct faults in recommendation algorithms. We\npropose a test suite that can be used to validate the correctness of a\nrecommendation algorithm, and thus identify and correct issues that can\naffect the performance and behavior of these algorithms. Our test suite\ncontains both black box and white box tests at every level of abstraction,\ni.e., system, integration and unit. To facilitate adoption, we release\nRecPack Tests, an open-source Python package containing template test\nimplementations. We use it to test four popular Python packages for\nrecommender systems: RecPack, PyLensKit, Surprise and Cornac. Despite the\nhigh test coverage of each of these packages, we find that we are still\nable to uncover undocumented functional requirements and even some bugs.\nThis validates our thesis that testing the correctness of recommendation\nalgorithms can complement traditional methods for evaluating\nrecommendation algorithms.},\n\tjournal = {ACM Trans. Recomm. Syst.},\n\tauthor = {Michiels, Lien and Verachtert, Robin and Ferraro, Andres and Falk, Kim and Goethals, Bart},\n\tmonth = apr,\n\tyear = {2023},\n\tnote = {Place: New York, NY, USA\nPublisher: Association for Computing Machinery},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Evaluating recommender systems adequately and thoroughly is an important task. Significant efforts are dedicated to proposing metrics, methods and protocols for doing so. However, there has been little discussion in the recommender systems’ literature on the topic of testing. In this work, we adopt and adapt concepts from the software testing domain, e.g., code coverage, metamorphic testing, or property-based testing, to help researchers to detect and correct faults in recommendation algorithms. We propose a test suite that can be used to validate the correctness of a recommendation algorithm, and thus identify and correct issues that can affect the performance and behavior of these algorithms. Our test suite contains both black box and white box tests at every level of abstraction, i.e., system, integration and unit. To facilitate adoption, we release RecPack Tests, an open-source Python package containing template test implementations. We use it to test four popular Python packages for recommender systems: RecPack, PyLensKit, Surprise and Cornac. Despite the high test coverage of each of these packages, we find that we are still able to uncover undocumented functional requirements and even some bugs. This validates our thesis that testing the correctness of recommendation algorithms can complement traditional methods for evaluating recommendation algorithms.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3604915.3610640\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023', 'https://dl.acm.org/doi/10.1145/3604915.3610640', event);\">\n    EasyStudy: Framework for Easy Deployment of User Studies on Recommender Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Dokoupil, P.;  and Peska, L.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 17th ACM Conference on Recommender Systems</i>, of <i>RecSys '23</i>, pages 1196–1199, New York, NY, USA, September 2023. Association for Computing Machinery\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3604915.3610640\"\n     onclick=\"log_download('dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023', 'https://dl.acm.org/doi/10.1145/3604915.3610640', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"EasyStudy: Framework for Easy Deployment of User Studies on Recommender Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3604915.3610640\"\n     onclick=\"log_download('dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023', 'http://doi.org/10.1145/3604915.3610640', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('dokoupil-peska-easystudyframeworkforeasydeploymentofuserstudiesonrecommendersystems-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{dokoupil_easystudy_2023,\n\taddress = {New York, NY, USA},\n\tseries = {{RecSys} &#x27;23},\n\ttitle = {{EasyStudy}: {Framework} for {Easy} {Deployment} of {User} {Studies} on {Recommender} {Systems}},\n\tisbn = {9798400702419},\n\tshorttitle = {{EasyStudy}},\n\turl = {https://dl.acm.org/doi/10.1145/3604915.3610640},\n\tdoi = {10.1145/3604915.3610640},\n\tabstract = {Improvements in the recommender systems (RS) domain are not possible without a thorough way to evaluate and compare newly proposed approaches. User studies represent a viable alternative to online and offline evaluation schemes, but despite their numerous benefits, they are only rarely used. One of the main reasons behind this fact is that preparing a user study from scratch involves a lot of extra work on top of a simple algorithm proposal. To simplify this task, we propose EasyStudy, a modular framework built on the credo “Make simple things fast and hard things possible”. It features ready-to-use datasets, preference elicitation methods, incrementally tuned baseline algorithms, study flow plugins, and evaluation metrics. As a result, a simple study comparing several RS can be deployed with just a few clicks, while more complex study designs can still benefit from a range of reusable components, such as preference elicitation. Overall, EasyStudy dramatically decreases the gap between the laboriousness of offline evaluation vs. user studies and, therefore, may contribute towards the more reliable and insightful user-centric evaluation of next-generation RS. The project repository is available from https://bit.ly/easy-study-repo.},\n\turldate = {2023-09-18},\n\tbooktitle = {Proceedings of the 17th {ACM} {Conference} on {Recommender} {Systems}},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Dokoupil, Patrik and Peska, Ladislav},\n\tmonth = sep,\n\tyear = {2023},\n\tkeywords = {Recommender systems, evaluation frameworks, user centric, user studies},\n\tpages = {1196--1199},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Improvements in the recommender systems (RS) domain are not possible without a thorough way to evaluate and compare newly proposed approaches. User studies represent a viable alternative to online and offline evaluation schemes, but despite their numerous benefits, they are only rarely used. One of the main reasons behind this fact is that preparing a user study from scratch involves a lot of extra work on top of a simple algorithm proposal. To simplify this task, we propose EasyStudy, a modular framework built on the credo “Make simple things fast and hard things possible”. It features ready-to-use datasets, preference elicitation methods, incrementally tuned baseline algorithms, study flow plugins, and evaluation metrics. As a result, a simple study comparing several RS can be deployed with just a few clicks, while more complex study designs can still benefit from a range of reusable components, such as preference elicitation. Overall, EasyStudy dramatically decreases the gap between the laboriousness of offline evaluation vs. user studies and, therefore, may contribute towards the more reliable and insightful user-centric evaluation of next-generation RS. The project repository is available from https://bit.ly/easy-study-repo.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3616088\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023', 'https://dl.acm.org/doi/10.1145/3616088', event);\">\n    Understanding the Contribution of Recommendation Algorithms on Misinformation Recommendation and Misinformation Dissemination on Social Networks.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Pathak, R.; Spezzano, F.;  and Pera, M. S.\n</span>\n\n  \n\n</span>\n\n  <i>ACM Transactions on the Web</i>. August 2023.\n  <span class=\"note\">Just Accepted</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3616088\"\n     onclick=\"log_download('pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023', 'https://dl.acm.org/doi/10.1145/3616088', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Understanding the Contribution of Recommendation Algorithms on Misinformation Recommendation and Misinformation Dissemination on Social Networks [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3616088\"\n     onclick=\"log_download('pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023', 'http://doi.org/10.1145/3616088', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('pathak-spezzano-pera-understandingthecontributionofrecommendationalgorithmsonmisinformationrecommendationandmisinformationdisseminationonsocialnetworks-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{pathak_understanding_2023,\n\ttitle = {Understanding the {Contribution} of {Recommendation} {Algorithms} on {Misinformation} {Recommendation} and {Misinformation} {Dissemination} on {Social} {Networks}},\n\tissn = {1559-1131},\n\turl = {https://dl.acm.org/doi/10.1145/3616088},\n\tdoi = {10.1145/3616088},\n\tabstract = {Social networks are a platform for individuals and organizations to connect with each other and inform, advertise, spread ideas, and ultimately influence opinions. These platforms have been known to propel misinformation. We argue that this could be compounded by the recommender algorithms that these platforms use to suggest items potentially of interest to their users, given the known biases and filter bubbles issues affecting recommender systems. While much has been studied about misinformation on social networks, the potential exacerbation that could result from recommender algorithms in this environment is in its infancy. In this manuscript, we present the result of an in-depth analysis conducted on two datasets (Politifact FakeNewsNet dataset and HealthStory FakeHealth dataset) in order to deepen our understanding of the interconnection between recommender algorithms and misinformation spread on Twitter. In particular, we explore the degree to which well-known recommendation algorithms are prone to be impacted by misinformation. Via simulation, we also study misinformation diffusion on social networks, as triggered by suggestions produced by these recommendation algorithms. Outcomes from this work evidence that misinformation does not equally affect all recommendation algorithms. Popularity-based and network-based recommender algorithms contribute the most to misinformation diffusion. Users who are known to be superspreaders are known to directly impact algorithmic performance and misinformation spread in specific scenarios. Findings emerging from our exploration result in a number of implications for researchers and practitioners to consider when designing and deploying recommender algorithms in social networks.},\n\turldate = {2023-09-19},\n\tjournal = {ACM Transactions on the Web},\n\tauthor = {Pathak, Royal and Spezzano, Francesca and Pera, Maria Soledad},\n\tmonth = aug,\n\tyear = {2023},\n\tnote = {Just Accepted},\n\tkeywords = {Twitter, diffusion, misinformation, news, recommendation algorithms, social networks},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Social networks are a platform for individuals and organizations to connect with each other and inform, advertise, spread ideas, and ultimately influence opinions. These platforms have been known to propel misinformation. We argue that this could be compounded by the recommender algorithms that these platforms use to suggest items potentially of interest to their users, given the known biases and filter bubbles issues affecting recommender systems. While much has been studied about misinformation on social networks, the potential exacerbation that could result from recommender algorithms in this environment is in its infancy. In this manuscript, we present the result of an in-depth analysis conducted on two datasets (Politifact FakeNewsNet dataset and HealthStory FakeHealth dataset) in order to deepen our understanding of the interconnection between recommender algorithms and misinformation spread on Twitter. In particular, we explore the degree to which well-known recommendation algorithms are prone to be impacted by misinformation. Via simulation, we also study misinformation diffusion on social networks, as triggered by suggestions produced by these recommendation algorithms. Outcomes from this work evidence that misinformation does not equally affect all recommendation algorithms. Popularity-based and network-based recommender algorithms contribute the most to misinformation diffusion. Users who are known to be superspreaders are known to directly impact algorithmic performance and misinformation spread in specific scenarios. Findings emerging from our exploration result in a number of implications for researchers and practitioners to consider when designing and deploying recommender algorithms in social networks.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023', 'https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7', event);\">\n    Measuring the effect of collaborative filtering on the diversity of users’ attention.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Godinot, A.;  and Tarissan, F.\n</span>\n\n  \n\n</span>\n\n  <i>Applied Network Science</i>, 8(1): 1–18. December 2023.\n  <span class=\"note\">Number: 1 Publisher: SpringerOpen</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7\"\n     onclick=\"log_download('godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023', 'https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Measuring the effect of collaborative filtering on the diversity of users’ attention [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s41109-022-00530-7\"\n     onclick=\"log_download('godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023', 'http://doi.org/10.1007/s41109-022-00530-7', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('godinot-tarissan-measuringtheeffectofcollaborativefilteringonthediversityofusersattention-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{godinot_measuring_2023,\n\ttitle = {Measuring the effect of collaborative filtering on the diversity of users’ attention},\n\tvolume = {8},\n\tcopyright = {2023 The Author(s)},\n\tissn = {2364-8228},\n\turl = {https://appliednetsci.springeropen.com/articles/10.1007/s41109-022-00530-7},\n\tdoi = {10.1007/s41109-022-00530-7},\n\tabstract = {While the ever-increasing emergence of online services has led to a growing interest in the development of recommender systems, the algorithms underpinning such systems have begun to be criticized for their role in limiting the variety of content exposed to users. In this context, the notion of diversity has been proposed as a way of mitigating the side effects resulting from the specialization of recommender systems. In this paper, using a well-known recommender system that makes use of collaborative filtering in the context of musical content, we analyze the diversity of recommendations generated through the lens of the recently proposed information network diversity measure. The results of our study offer significant insights into the effect of algorithmic recommendations. On the one hand, we show that the musical selections of a large proportion of users are diversified as a result of the recommendations. On the other hand, however, such improvements do not benefit all users. They are in fact mainly restricted to users with a low level of activity or whose past musical listening selections are very narrow. Through more in-depth investigations, we also discovered that while recommendations generally increase the variety of the songs recommended to users, they nonetheless fail to provide a balanced exposure to the different related categories.},\n\tlanguage = {en},\n\tnumber = {1},\n\turldate = {2023-09-19},\n\tjournal = {Applied Network Science},\n\tauthor = {Godinot, Augustin and Tarissan, Fabien},\n\tmonth = dec,\n\tyear = {2023},\n\tnote = {Number: 1\nPublisher: SpringerOpen},\n\tpages = {1--18},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  While the ever-increasing emergence of online services has led to a growing interest in the development of recommender systems, the algorithms underpinning such systems have begun to be criticized for their role in limiting the variety of content exposed to users. In this context, the notion of diversity has been proposed as a way of mitigating the side effects resulting from the specialization of recommender systems. In this paper, using a well-known recommender system that makes use of collaborative filtering in the context of musical content, we analyze the diversity of recommendations generated through the lens of the recently proposed information network diversity measure. The results of our study offer significant insights into the effect of algorithmic recommendations. On the one hand, we show that the musical selections of a large proportion of users are diversified as a result of the recommendations. On the other hand, however, such improvements do not benefit all users. They are in fact mainly restricted to users with a low level of activity or whose past musical listening selections are very narrow. Through more in-depth investigations, we also discovered that while recommendations generally increase the variety of the songs recommended to users, they nonetheless fail to provide a balanced exposure to the different related categories.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3563359.3597386\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023', 'https://dl.acm.org/doi/10.1145/3563359.3597386', event);\">\n    The Effect of Similarity Metric and Group Size on Outlier Selection & Satisfaction in Group Recommender Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Dokoupil, P.;  and Peska, L.\n</span>\n\n  \n\n</span>\n\n  In <i>Adjunct Proceedings of the 31st ACM Conference on User Modeling, Adaptation and Personalization</i>, of <i>UMAP '23 Adjunct</i>, pages 296–301, New York, NY, USA, June 2023. Association for Computing Machinery\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://dl.acm.org/doi/10.1145/3563359.3597386\"\n     onclick=\"log_download('dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023', 'https://dl.acm.org/doi/10.1145/3563359.3597386', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"The Effect of Similarity Metric and Group Size on Outlier Selection &amp; Satisfaction in Group Recommender Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3563359.3597386\"\n     onclick=\"log_download('dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023', 'http://doi.org/10.1145/3563359.3597386', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('dokoupil-peska-theeffectofsimilaritymetricandgroupsizeonoutlierselectionsatisfactioningrouprecommendersystems-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{dokoupil_effect_2023,\n\taddress = {New York, NY, USA},\n\tseries = {{UMAP} &#x27;23 {Adjunct}},\n\ttitle = {The {Effect} of {Similarity} {Metric} and {Group} {Size} on {Outlier} {Selection} \\&amp; {Satisfaction} in {Group} {Recommender} {Systems}},\n\tisbn = {978-1-4503-9891-6},\n\turl = {https://dl.acm.org/doi/10.1145/3563359.3597386},\n\tdoi = {10.1145/3563359.3597386},\n\tabstract = {Group recommender systems (GRS) are a specific case of recommender systems (RS), where recommendations are constructed to a group of users rather than an individual. GRS has diverse application areas including trip planning, recommending movies to watch together, or music in shared environments. However, due to the lack of large datasets with group decision-making feedback information, or even the group definitions, GRS approaches are often evaluated offline w.r.t. individual user feedback and artificially generated groups. These synthetic groups are usually constructed w.r.t. pre-defined group size and inter-user similarity metric. While numerous variants of synthetic group generation procedures were utilized so far, its impact on the evaluation results was not sufficiently discussed. In this paper, we address this research gap by investigating the impact of various synthetic group generation procedures, namely the usage of different user similarity metrics and the effect of group sizes. We consider them in the context of “outlier vs. majority” groups, where a group of similar users is extended with one or more diverse ones. Experimental results indicate a strong impact of the selected similarity metric on both the typical characteristics of selected outliers as well as the performance of individual GRS algorithms. Moreover, we show that certain algorithms better adapt to larger groups than others.},\n\turldate = {2023-06-25},\n\tbooktitle = {Adjunct {Proceedings} of the 31st {ACM} {Conference} on {User} {Modeling}, {Adaptation} and {Personalization}},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Dokoupil, Patrik and Peska, Ladislav},\n\tmonth = jun,\n\tyear = {2023},\n\tkeywords = {Group Recommender systems, Synthetic groups construction, User similarity metrics},\n\tpages = {296--301},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Group recommender systems (GRS) are a specific case of recommender systems (RS), where recommendations are constructed to a group of users rather than an individual. GRS has diverse application areas including trip planning, recommending movies to watch together, or music in shared environments. However, due to the lack of large datasets with group decision-making feedback information, or even the group definitions, GRS approaches are often evaluated offline w.r.t. individual user feedback and artificially generated groups. These synthetic groups are usually constructed w.r.t. pre-defined group size and inter-user similarity metric. While numerous variants of synthetic group generation procedures were utilized so far, its impact on the evaluation results was not sufficiently discussed. In this paper, we address this research gap by investigating the impact of various synthetic group generation procedures, namely the usage of different user similarity metrics and the effect of group sizes. We consider them in the context of “outlier vs. majority” groups, where a group of similar users is extended with one or more diverse ones. Experimental results indicate a strong impact of the selected similarity metric on both the typical characteristics of selected outliers as well as the performance of individual GRS algorithms. Moreover, we show that certain algorithms better adapt to larger groups than others.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3576840.3578274\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023', 'https://doi.org/10.1145/3576840.3578274', event);\">\n    FairRecKit: A Web-based Analysis Software for Recommender Evaluations.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Bauer, C.; Chung, L.; Cornelissen, A.; van Driessel, I.; van der Hoorn, D.; de Jong, Y.; Le, L.; Najiyan Tabriz, S.; Spaans, R.; Thijsen, C.; Verbeeten, R.; Wesseling, V.;  and Wieland, F.\n</span>\n\n  \n\n</span>\n\n  In <i>CHIIR '23</i>, pages 438–443, New York, NY, USA, March 2023. Association for Computing Machinery\n  <span class=\"note\">Journal Abbreviation: CHIIR '23</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3576840.3578274\"\n     onclick=\"log_download('bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023', 'https://doi.org/10.1145/3576840.3578274', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"FairRecKit: A Web-based Analysis Software for Recommender Evaluations [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3576840.3578274\"\n     onclick=\"log_download('bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023', 'http://doi.org/10.1145/3576840.3578274', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>5 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('bauer-chung-cornelissen-vandriessel-vanderhoorn-dejong-le-najiyantabriz-etal-fairreckitawebbasedanalysissoftwareforrecommenderevaluations-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{bauer_fairreckit_2023,\n\taddress = {New York, NY, USA},\n\ttitle = {{FairRecKit}: {A} {Web}-based {Analysis} {Software} for {Recommender} {Evaluations}},\n\turl = {https://doi.org/10.1145/3576840.3578274},\n\tdoi = {10.1145/3576840.3578274},\n\tabstract = {FairRecKit is a web-based analysis software that supports researchers in\nperforming, analyzing, and understanding recommendation computations. The\nidea behind FairRecKit is to facilitate the in-depth analysis of\nrecommendation outcomes considering fairness aspects. With (nested)\nfilters on user or item attributes, metrics can easily be compared across\nuser and item subgroups. Further, (nested) filters can be used on the\ndataset level; this way, recommendation outcomes can be compared across\nseveral sub-datasets to analyze for differences considering fairness\naspects. The software currently features five datasets, 11 metrics, and 21\nrecommendation algorithms to be used in computational experimentation. It\nis open source and developed in a modular manner to facilitate extension.\nThe analysis software consists of two components: A software package\n(FairRecKitLib) for running recommendation algorithms on the available\ndatasets and a web-based user interface (FairRecKitApp) to start\nexperiments, retrieve results of previous experiments, and analyze\ndetails. The application also comes with extensive documentation and\noptions for result customization, which makes for a flexible tool that\nsupports in-depth analysis.},\n\turldate = {2023-03-28},\n\tbooktitle = {{CHIIR} &#x27;23},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Bauer, Christine and Chung, Lennard and Cornelissen, Aleksej and van Driessel, Isabelle and van der Hoorn, Diede and de Jong, Yme and Le, Lan and Najiyan Tabriz, Sanaz and Spaans, Roderick and Thijsen, Casper and Verbeeten, Robert and Wesseling, Vos and Wieland, Fern},\n\tmonth = mar,\n\tyear = {2023},\n\tnote = {Journal Abbreviation: CHIIR &#x27;23},\n\tkeywords = {evaluation, music, web-based, recommender systems, toolkit, analysis, movies, resource},\n\tpages = {438--443},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  FairRecKit is a web-based analysis software that supports researchers in performing, analyzing, and understanding recommendation computations. The idea behind FairRecKit is to facilitate the in-depth analysis of recommendation outcomes considering fairness aspects. With (nested) filters on user or item attributes, metrics can easily be compared across user and item subgroups. Further, (nested) filters can be used on the dataset level; this way, recommendation outcomes can be compared across several sub-datasets to analyze for differences considering fairness aspects. The software currently features five datasets, 11 metrics, and 21 recommendation algorithms to be used in computational experimentation. It is open source and developed in a modular manner to facilitate extension. The analysis software consists of two components: A software package (FairRecKitLib) for running recommendation algorithms on the available datasets and a web-based user interface (FairRecKitApp) to start experiments, retrieve results of previous experiments, and analyze details. The application also comes with extensive documentation and options for result customization, which makes for a flexible tool that supports in-depth analysis.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.mdpi.com/1099-4300/25/1/132\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023', 'https://www.mdpi.com/1099-4300/25/1/132', event);\">\n    A Comparative Study of Rank Aggregation Methods in Recommendation Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Bałchanowski, M.;  and Boryczka, U.\n</span>\n\n  \n\n</span>\n\n  <i>Entropy</i>, 25(1): 132. January 2023.\n  <span class=\"note\">Publisher: Multidisciplinary Digital Publishing Institute</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.mdpi.com/1099-4300/25/1/132\"\n     onclick=\"log_download('bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023', 'https://www.mdpi.com/1099-4300/25/1/132', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"A Comparative Study of Rank Aggregation Methods in Recommendation Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.3390/e25010132\"\n     onclick=\"log_download('bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023', 'http://doi.org/10.3390/e25010132', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('bachanowski-boryczka-acomparativestudyofrankaggregationmethodsinrecommendationsystems-2023')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{balchanowski_comparative_2023,\n\ttitle = {A {Comparative} {Study} of {Rank} {Aggregation} {Methods} in {Recommendation} {Systems}},\n\tvolume = {25},\n\tissn = {1099-4300},\n\turl = {https://www.mdpi.com/1099-4300/25/1/132},\n\tdoi = {10.3390/e25010132},\n\tabstract = {The aim of a recommender system is to suggest to the user certain products\nor services that most likely will interest them. Within the context of\npersonalized recommender systems, a number of algorithms have been\nsuggested to generate a ranking of items tailored to individual user\npreferences. However, these algorithms do not generate identical\nrecommendations, and for this reason it has been suggested in the\nliterature that the results of these algorithms can be combined using\naggregation techniques, hoping that this will translate into an\nimprovement in the quality of the final recommendation. In order to see\nwhich of these techniques increase the quality of recommendations to the\ngreatest extent, the authors of this publication conducted experiments in\nwhich they considered five recommendation algorithms and 20 aggregation\nmethods. The research was carried out on the popular and publicly\navailable MovieLens 100k and MovieLens 1M datasets, and the results were\nconfirmed by statistical tests.},\n\tnumber = {1},\n\turldate = {2023-01-12},\n\tjournal = {Entropy},\n\tauthor = {Bałchanowski, Michał and Boryczka, Urszula},\n\tmonth = jan,\n\tyear = {2023},\n\tnote = {Publisher: Multidisciplinary Digital Publishing Institute},\n\tpages = {132},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  The aim of a recommender system is to suggest to the user certain products or services that most likely will interest them. Within the context of personalized recommender systems, a number of algorithms have been suggested to generate a ranking of items tailored to individual user preferences. However, these algorithms do not generate identical recommendations, and for this reason it has been suggested in the literature that the results of these algorithms can be combined using aggregation techniques, hoping that this will translate into an improvement in the quality of the final recommendation. In order to see which of these techniques increase the quality of recommendations to the greatest extent, the authors of this publication conducted experiments in which they considered five recommendation algorithms and 20 aggregation methods. The research was carried out on the popular and publicly available MovieLens 100k and MovieLens 1M datasets, and the results were confirmed by statistical tests.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2022\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2022')\"\n      onkeypress=\"toggleGroup('group_2022')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2022</span>\n      <span class=\"bibbase_group_count\">\n        \n        (10)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/fair-ranking\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'https://md.ekstrandom.net/pubs/fair-ranking', event);\">\n    Measuring fairness in ranked results: an analytical and empirical comparison.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.;  and Ekstrand, M. D\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval</i>, pages 726–736, July 2022. ACM\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/fair-ranking\"\n     onclick=\"log_download('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'https://md.ekstrandom.net/pubs/fair-ranking', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Measuring fairness in ranked results: an analytical and empirical comparison [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3477495.3532018\"\n     onclick=\"log_download('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022', 'http://doi.org/10.1145/3477495.3532018', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>4 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-ekstrand-measuringfairnessinrankedresultsananalyticalandempiricalcomparison-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_measuring_2022,\n\ttitle = {Measuring fairness in ranked results: an analytical and empirical comparison},\n\turl = {https://md.ekstrandom.net/pubs/fair-ranking},\n\tdoi = {10.1145/3477495.3532018},\n\tabstract = {Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user&#x27;s information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system&#x27;s behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking.\n\nWe aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.},\n\tbooktitle = {Proceedings of the 45th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},\n\tpublisher = {ACM},\n\tauthor = {Raj, Amifa and Ekstrand, Michael D},\n\tmonth = jul,\n\tyear = {2022},\n\tpages = {726--736},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Information access systems, such as search and recommender systems, often use ranked lists to present results believed to be relevant to the user's information need. Evaluating these lists for their fairness along with other traditional metrics provides a more complete understanding of an information access system's behavior beyond accuracy or utility constructs. To measure the (un)fairness of rankings, particularly with respect to the protected group(s) of producers or providers, several metrics have been proposed in the last several years. However, an empirical and comparative analyses of these metrics showing the applicability to specific scenario or real data, conceptual similarities, and differences is still lacking. We aim to bridge the gap between theoretical and practical ap-plication of these metrics. In this paper we describe several fair ranking metrics from the existing literature in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data sets in the context of three information access tasks. We also provide a sensitivity analysis to assess the impact of the design choices and parameter settings that go in to these metrics and point to additional work needed to improve fairness measurement.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2212.13897\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022', 'http://arxiv.org/abs/2212.13897', event);\">\n    What You Like: Generating Explainable Topical Recommendations for Twitter Using Social Annotations.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Bhattacharya, P.; Ghosh, S.; Zafar, M. B.; Ghosh, S. K;  and Ganguly, N.\n</span>\n\n  \n\n</span>\n\n  December 2022.\n  <span class=\"note\">ISBN: 2212.13897 Publication Title: arXiv [cs.IR]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2212.13897\"\n     onclick=\"log_download('bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022', 'http://arxiv.org/abs/2212.13897', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"What You Like: Generating Explainable Topical Recommendations for Twitter Using Social Annotations [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('bhattacharya-ghosh-zafar-ghosh-ganguly-whatyoulikegeneratingexplainabletopicalrecommendationsfortwitterusingsocialannotations-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{bhattacharya_what_2022,\n\ttitle = {What {You} {Like}: {Generating} {Explainable} {Topical} {Recommendations} for {Twitter} {Using} {Social} {Annotations}},\n\turl = {http://arxiv.org/abs/2212.13897},\n\tabstract = {With over 500 million tweets posted per day, in Twitter, it is difficult\nfor Twitter users to discover interesting content from the deluge of\nuninteresting posts. In this work, we present a novel, explainable,\ntopical recommendation system, that utilizes social annotations, to help\nTwitter users discover tweets, on topics of their interest. A major\nchallenge in using traditional rating dependent recommendation systems,\nlike collaborative filtering and content based systems, in high volume\nsocial networks is that, due to attention scarcity most items do not get\nany ratings. Additionally, the fact that most Twitter users are passive\nconsumers, with 44\\% users never tweeting, makes it very difficult to use\nuser ratings for generating recommendations. Further, a key challenge in\ndeveloping recommendation systems is that in many cases users reject\nrelevant recommendations if they are totally unfamiliar with the\nrecommended item. Providing a suitable explanation, for why the item is\nrecommended, significantly improves the acceptability of recommendation.\nBy virtue of being a topical recommendation system our method is able to\npresent simple topical explanations for the generated recommendations.\nComparisons with state-of-the-art matrix factorization based collaborative\nfiltering, content based and social recommendations demonstrate the\nefficacy of the proposed approach.},\n\tauthor = {Bhattacharya, Parantapa and Ghosh, Saptarshi and Zafar, Muhammad Bilal and Ghosh, Soumya K and Ganguly, Niloy},\n\tmonth = dec,\n\tyear = {2022},\n\tnote = {ISBN: 2212.13897\nPublication Title: arXiv [cs.IR]},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  With over 500 million tweets posted per day, in Twitter, it is difficult for Twitter users to discover interesting content from the deluge of uninteresting posts. In this work, we present a novel, explainable, topical recommendation system, that utilizes social annotations, to help Twitter users discover tweets, on topics of their interest. A major challenge in using traditional rating dependent recommendation systems, like collaborative filtering and content based systems, in high volume social networks is that, due to attention scarcity most items do not get any ratings. Additionally, the fact that most Twitter users are passive consumers, with 44% users never tweeting, makes it very difficult to use user ratings for generating recommendations. Further, a key challenge in developing recommendation systems is that in many cases users reject relevant recommendations if they are totally unfamiliar with the recommended item. Providing a suitable explanation, for why the item is recommended, significantly improves the acceptability of recommendation. By virtue of being a topical recommendation system our method is able to present simple topical explanations for the generated recommendations. Comparisons with state-of-the-art matrix factorization based collaborative filtering, content based and social recommendations demonstrate the efficacy of the proposed approach.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://papers.ssrn.com/abstract=4274780\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022', 'https://papers.ssrn.com/abstract=4274780', event);\">\n    Recommendations with Minimum Exposure Guarantees: A Post-Processing Framework.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Lopes, R.; Alves, R.; Ledent, A.; Santos, R.;  and Kloft, M.\n</span>\n\n  \n\n</span>\n\n  November 2022.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://papers.ssrn.com/abstract=4274780\"\n     onclick=\"log_download('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022', 'https://papers.ssrn.com/abstract=4274780', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Recommendations with Minimum Exposure Guarantees: A Post-Processing Framework [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.2139/ssrn.4274780\"\n     onclick=\"log_download('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022', 'http://doi.org/10.2139/ssrn.4274780', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('lopes-alves-ledent-santos-kloft-recommendationswithminimumexposureguaranteesapostprocessingframework-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{lopes_recommendations_2022,\n\ttitle = {Recommendations with {Minimum} {Exposure} {Guarantees}: {A} {Post}-{Processing} {Framework}},\n\turl = {https://papers.ssrn.com/abstract=4274780},\n\tabstract = {Relevance-based ranking is a popular ingredient in recommenders, but it\nfrequently struggles to meet fairness criteria because social and cultural\nnorms may favor some item groups over others. For instance, some items\nmight receive lower ratings due to some sort of bias (e.g. gender bias). A\nfair ranking should balance the exposure of items from advantaged and\ndisadvantagedgroups. To this end, we propose a novel post-processing\nframework to produce fair, exposure-aware recommendations. Our approach is\nbased on an integer linear programming model maximizing the expected\nutility while satisfying a minimum exposure constraint. The model has\nfewer variables than previous work and thus can be deployed to larger\ndatasets and allows the organization to define a minimum level of exposure\nfor groups of items. We conduct an extensive empirical evaluation\nindicating that our new framework can increase the exposure of items from\ndisadvantaged groups at a small cost of recommendation accuracy.},\n\turldate = {2022-12-23},\n\tauthor = {Lopes, Ramon and Alves, Rodrigo and Ledent, Antoine and Santos, Rodrygo and Kloft, Marius},\n\tmonth = nov,\n\tyear = {2022},\n\tdoi = {10.2139/ssrn.4274780},\n\tkeywords = {recommender systems, fairness, exposure, integer linear programming},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Relevance-based ranking is a popular ingredient in recommenders, but it frequently struggles to meet fairness criteria because social and cultural norms may favor some item groups over others. For instance, some items might receive lower ratings due to some sort of bias (e.g. gender bias). A fair ranking should balance the exposure of items from advantaged and disadvantagedgroups. To this end, we propose a novel post-processing framework to produce fair, exposure-aware recommendations. Our approach is based on an integer linear programming model maximizing the expected utility while satisfying a minimum exposure constraint. The model has fewer variables than previous work and thus can be deployed to larger datasets and allows the organization to define a minimum level of exposure for groups of items. We conduct an extensive empirical evaluation indicating that our new framework can increase the exposure of items from disadvantaged groups at a small cost of recommendation accuracy.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S187705092201167X\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022', 'https://www.sciencedirect.com/science/article/pii/S187705092201167X', event);\">\n    Collaborative Rank Aggregation in Recommendation Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Bałchanowski, M.;  and Boryczka, U.\n</span>\n\n  \n\n</span>\n\n  <i>Procedia Comput. Sci.</i>, 207: 2213–2222. January 2022.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S187705092201167X\"\n     onclick=\"log_download('bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022', 'https://www.sciencedirect.com/science/article/pii/S187705092201167X', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Collaborative Rank Aggregation in Recommendation Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1016/j.procs.2022.09.281\"\n     onclick=\"log_download('bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022', 'http://doi.org/10.1016/j.procs.2022.09.281', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('bachanowski-boryczka-collaborativerankaggregationinrecommendationsystems-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{balchanowski_collaborative_2022,\n\ttitle = {Collaborative {Rank} {Aggregation} in {Recommendation} {Systems}},\n\tvolume = {207},\n\tissn = {1877-0509},\n\turl = {https://www.sciencedirect.com/science/article/pii/S187705092201167X},\n\tdoi = {10.1016/j.procs.2022.09.281},\n\tabstract = {Over the years, various techniques of generating recommendations have been\ndeveloped. However, it turns out that when we compare the recommendations\ngenerated by different algorithms in the context of a particular user, the\nquality of such recommendations for different techniques may differ. The\nuse of the aggregation techniques, the aim of which is to combine several\nrankings into one, can be a solution to this problem. In theory it should\nimprove the quality of the recommendations. Additionally, in order to\npersonalize the recommendations better, a metaheuristic algorithm, which,\nby assigning different weights to each feature, tries to represent the\npreference of the active user, was used. This paper also presents a\nsuggestion to include additional rankings generated for other users in the\nsystem in the aggregation process. The idea will be supported by research\nresults that clearly show that taking into account rankings of other users\ncan improve the quality of the generated recommendations.},\n\tjournal = {Procedia Comput. Sci.},\n\tauthor = {Bałchanowski, Michał and Boryczka, Urszula},\n\tmonth = jan,\n\tyear = {2022},\n\tkeywords = {differential evolution, metaheuristic, rank aggregation, recommendation systems},\n\tpages = {2213--2222},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Over the years, various techniques of generating recommendations have been developed. However, it turns out that when we compare the recommendations generated by different algorithms in the context of a particular user, the quality of such recommendations for different techniques may differ. The use of the aggregation techniques, the aim of which is to combine several rankings into one, can be a solution to this problem. In theory it should improve the quality of the recommendations. Additionally, in order to personalize the recommendations better, a metaheuristic algorithm, which, by assigning different weights to each feature, tries to represent the preference of the active user, was used. This paper also presents a suggestion to include additional rankings generated for other users in the system in the aggregation process. The idea will be supported by research results that clearly show that taking into account rankings of other users can improve the quality of the generated recommendations.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://ceur-ws.org/Vol-3228/paper2.pdf\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022', 'https://ceur-ws.org/Vol-3228/paper2.pdf', event);\">\n    CaMeLS: Cooperative meta-learning service for recommender systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wegmeth, L.;  and Beel, J.\n</span>\n\n  \n\n</span>\n\n  In September 2022. CEUR-WS\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://ceur-ws.org/Vol-3228/paper2.pdf\"\n     onclick=\"log_download('wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022', 'https://ceur-ws.org/Vol-3228/paper2.pdf', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/pdf.svg\"\n\t     alt=\"CaMeLS: Cooperative meta-learning service for recommender systems [pdf]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wegmeth-beel-camelscooperativemetalearningserviceforrecommendersystems-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{wegmeth_camels_2022,\n\ttitle = {{CaMeLS}: {Cooperative} meta-learning service for recommender systems},\n\turl = {https://ceur-ws.org/Vol-3228/paper2.pdf},\n\tabstract = {We present CaMeLS, a proof of concept of a cooperative meta-learning\nservice for recommender systems. CaMeLS leverages the computing power of\nrecommender systems users by uploading their metadata and algorithm\nevaluation scores to a centralized environment. Through the resulting\ndatabase, CaMeLS then offers meta-learning services for everyone.\nAdditionally, users may access evaluations of common data sets immediately\nto know the best-performing algorithms for those data sets. The metadata\ntable may also be used for other purposes, e.g., to perform benchmarks. In\nthe initial version discussed in this paper, CaMeLS implements automatic\nalgorithm selection through meta-learning over two recommender systems\nlibraries. Automatic algorithm selection saves users time and computing\npower and does not require expertise, as the best algorithm is\nautomatically found over multiple libraries. The CaMeLS database contains\n20 metadata sets by default. We show that the automatic algorithm\nselection service is already on par with the single best algorithm in this\ndefault scenario. CaMeLS only requires a few seconds to predict a suitable\nalgorithm, rather than potentially hours or days if performed manually,\ndepending on the data set. The code is publicly available on our GitHub\nhttps://camels.recommender-systems.com.},\n\turldate = {2022-11-13},\n\tpublisher = {CEUR-WS},\n\tauthor = {Wegmeth, Lukas and Beel, Joeran},\n\tmonth = sep,\n\tyear = {2022},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  We present CaMeLS, a proof of concept of a cooperative meta-learning service for recommender systems. CaMeLS leverages the computing power of recommender systems users by uploading their metadata and algorithm evaluation scores to a centralized environment. Through the resulting database, CaMeLS then offers meta-learning services for everyone. Additionally, users may access evaluations of common data sets immediately to know the best-performing algorithms for those data sets. The metadata table may also be used for other purposes, e.g., to perform benchmarks. In the initial version discussed in this paper, CaMeLS implements automatic algorithm selection through meta-learning over two recommender systems libraries. Automatic algorithm selection saves users time and computing power and does not require expertise, as the best algorithm is automatically found over multiple libraries. The CaMeLS database contains 20 metadata sets by default. We show that the automatic algorithm selection service is already on par with the single best algorithm in this default scenario. CaMeLS only requires a few seconds to predict a suitable algorithm, rather than potentially hours or days if performed manually, depending on the data set. The code is publicly available on our GitHub https://camels.recommender-systems.com.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2211.15608\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022', 'http://arxiv.org/abs/2211.15608', event);\">\n    Representation with Incomplete Votes.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Halpern, D.; Kehne, G.; Procaccia, A. D; Tucker-Foltz, J.;  and Wüthrich, M.\n</span>\n\n  \n\n</span>\n\n  November 2022.\n  <span class=\"note\">ISBN: 2211.15608 Publication Title: arXiv [cs.GT]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2211.15608\"\n     onclick=\"log_download('halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022', 'http://arxiv.org/abs/2211.15608', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Representation with Incomplete Votes [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('halpern-kehne-procaccia-tuckerfoltz-wthrich-representationwithincompletevotes-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{halpern_representation_2022,\n\ttitle = {Representation with {Incomplete} {Votes}},\n\turl = {http://arxiv.org/abs/2211.15608},\n\tabstract = {Platforms for online civic participation rely heavily on methods for\ncondensing thousands of comments into a relevant handful, based on whether\nparticipants agree or disagree with them. These methods should guarantee\nfair representation of the participants, as their outcomes may affect the\nhealth of the conversation and inform impactful downstream decisions. To\nthat end, we draw on the literature on approval-based committee elections.\nOur setting is novel in that the approval votes are incomplete since\nparticipants will typically not vote on all comments. We prove that this\ncomplication renders non-adaptive algorithms impractical in terms of the\namount of information they must gather. Therefore, we develop an adaptive\nalgorithm that uses information more efficiently by presenting incoming\nparticipants with statements that appear promising based on votes by\nprevious participants. We prove that this method satisfies commonly used\nnotions of fair representation, even when participants only vote on a\nsmall fraction of comments. Finally, an empirical evaluation using real\ndata shows that the proposed algorithm provides representative outcomes in\npractice.},\n\tauthor = {Halpern, Daniel and Kehne, Gregory and Procaccia, Ariel D and Tucker-Foltz, Jamie and Wüthrich, Manuel},\n\tmonth = nov,\n\tyear = {2022},\n\tnote = {ISBN: 2211.15608\nPublication Title: arXiv [cs.GT]},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Platforms for online civic participation rely heavily on methods for condensing thousands of comments into a relevant handful, based on whether participants agree or disagree with them. These methods should guarantee fair representation of the participants, as their outcomes may affect the health of the conversation and inform impactful downstream decisions. To that end, we draw on the literature on approval-based committee elections. Our setting is novel in that the approval votes are incomplete since participants will typically not vote on all comments. We prove that this complication renders non-adaptive algorithms impractical in terms of the amount of information they must gather. Therefore, we develop an adaptive algorithm that uses information more efficiently by presenting incoming participants with statements that appear promising based on votes by previous participants. We prove that this method satisfies commonly used notions of fair representation, even when participants only vote on a small fraction of comments. Finally, an empirical evaluation using real data shows that the proposed algorithm provides representative outcomes in practice.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3565480\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022', 'https://doi.org/10.1145/3565480', event);\">\n    User Perception of Recommendation Explanation: Are Your Explanations What Users Need?.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Lu, H.; Ma, W.; Wang, Y.; Zhang, M.; Wang, X.; Liu, Y.; Chua, T.;  and Ma, S.\n</span>\n\n  \n\n</span>\n\n  <i>ACM Trans. Inf. Syst. Secur.</i>. November 2022.\n  <span class=\"note\">Place: New York, NY, USA Publisher: Association for Computing Machinery</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3565480\"\n     onclick=\"log_download('lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022', 'https://doi.org/10.1145/3565480', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"User Perception of Recommendation Explanation: Are Your Explanations What Users Need? [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3565480\"\n     onclick=\"log_download('lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022', 'http://doi.org/10.1145/3565480', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>3 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('lu-ma-wang-zhang-wang-liu-chua-ma-userperceptionofrecommendationexplanationareyourexplanationswhatusersneed-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{lu_user_2022,\n\ttitle = {User {Perception} of {Recommendation} {Explanation}: {Are} {Your} {Explanations} {What} {Users} {Need}?},\n\tissn = {1094-9224},\n\turl = {https://doi.org/10.1145/3565480},\n\tdoi = {10.1145/3565480},\n\tabstract = {As recommender systems become increasingly important in daily human\ndecision-making, users are demanding convincing explanations to understand\nthey get the specific recommendation results. Although a number of\nexplainable recommender systems have recently been proposed, there still\nlacks an understanding of what users really need in a recommendation\nexplanation. The actual reason behind users’ intention to examine and\nconsume (e.g., click and watch a movie) can be the window to answer this\nquestion and is named as self-explanation in this work. In addition,\nhumans usually make recommendations accompanied by explanations, but there\nremain fewer studies on how humans explain and what we can learn from\nhuman-generated explanations. To investigate these questions, we conduct a\nnovel multi-role, multi-session user study in which users interact with\nmultiple types of system-generated explanations as well as human-generated\nexplanations, namely peer-explanation. During the study, users’\nintentions, expectations, and experiences are tracked in several phases,\nincluding before and after the users are presented with an explanation and\nafter the content is examined. Through comprehensive investigations, three\nmain findings have been made: First, we observe not only the positive but\nalso the negative effects of explanations, and the impact varies across\ndifferent types of explanations. Moreover, human-generated explanation,\npeer-explanation, performs better in increasing user intentions and\nhelping users to better construct preferences, which results in better\nuser satisfaction. Second, based on users’ self-explanation, the\ninformation accuracy is measured and found to be a major factor associated\nwith user satisfaction. Some other factors, such as unfamiliarity and\nsimilarity, are also discovered and summarized. Third, through annotations\nof the information aspects used in the human-generated self-explanation\nand peer-explanation, patterns of how humans explain are investigated,\nincluding what information and how much information is utilized. In\naddition, based on the findings, a human-inspired explanation approach is\nproposed and found to increase user satisfaction, revealing the potential\nimprovement of further incorporating more human patterns in recommendation\nexplanations. These findings have shed light on the deeper understanding\nof the recommendation explanation and further research on its evaluation\nand generation. Furthermore, the collected data, including human-generated\nexplanations by both the external peers and the users’ selves, will be\nreleased to support future research works on explanation evaluation.},\n\tjournal = {ACM Trans. Inf. Syst. Secur.},\n\tauthor = {Lu, Hongyu and Ma, Weizhi and Wang, Yifan and Zhang, Min and Wang, Xiang and Liu, Yiqun and Chua, Tat-Seng and Ma, Shaoping},\n\tmonth = nov,\n\tyear = {2022},\n\tnote = {Place: New York, NY, USA\nPublisher: Association for Computing Machinery},\n\tkeywords = {Recommendation Explanation, User Modeling, Recommender System},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  As recommender systems become increasingly important in daily human decision-making, users are demanding convincing explanations to understand they get the specific recommendation results. Although a number of explainable recommender systems have recently been proposed, there still lacks an understanding of what users really need in a recommendation explanation. The actual reason behind users’ intention to examine and consume (e.g., click and watch a movie) can be the window to answer this question and is named as self-explanation in this work. In addition, humans usually make recommendations accompanied by explanations, but there remain fewer studies on how humans explain and what we can learn from human-generated explanations. To investigate these questions, we conduct a novel multi-role, multi-session user study in which users interact with multiple types of system-generated explanations as well as human-generated explanations, namely peer-explanation. During the study, users’ intentions, expectations, and experiences are tracked in several phases, including before and after the users are presented with an explanation and after the content is examined. Through comprehensive investigations, three main findings have been made: First, we observe not only the positive but also the negative effects of explanations, and the impact varies across different types of explanations. Moreover, human-generated explanation, peer-explanation, performs better in increasing user intentions and helping users to better construct preferences, which results in better user satisfaction. Second, based on users’ self-explanation, the information accuracy is measured and found to be a major factor associated with user satisfaction. Some other factors, such as unfamiliarity and similarity, are also discovered and summarized. Third, through annotations of the information aspects used in the human-generated self-explanation and peer-explanation, patterns of how humans explain are investigated, including what information and how much information is utilized. In addition, based on the findings, a human-inspired explanation approach is proposed and found to increase user satisfaction, revealing the potential improvement of further incorporating more human patterns in recommendation explanations. These findings have shed light on the deeper understanding of the recommendation explanation and further research on its evaluation and generation. Furthermore, the collected data, including human-generated explanations by both the external peers and the users’ selves, will be released to support future research works on explanation evaluation.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.hindawi.com/journals/mpe/2022/2340671/\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022', 'https://www.hindawi.com/journals/mpe/2022/2340671/', event);\">\n    A New Collaborative Filtering Algorithm Integrating Time and Multisimilarity.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Liu, Q.\n</span>\n\n  \n\n</span>\n\n  <i>Math. Probl. Eng.</i>, 2022. August 2022.\n  <span class=\"note\">Publisher: Hindawi</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.hindawi.com/journals/mpe/2022/2340671/\"\n     onclick=\"log_download('liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022', 'https://www.hindawi.com/journals/mpe/2022/2340671/', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"A New Collaborative Filtering Algorithm Integrating Time and Multisimilarity [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1155/2022/2340671\"\n     onclick=\"log_download('liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022', 'http://doi.org/10.1155/2022/2340671', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>3 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('liu-anewcollaborativefilteringalgorithmintegratingtimeandmultisimilarity-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{liu_new_2022,\n\ttitle = {A {New} {Collaborative} {Filtering} {Algorithm} {Integrating} {Time} and {Multisimilarity}},\n\tvolume = {2022},\n\tissn = {1024-123X},\n\turl = {https://www.hindawi.com/journals/mpe/2022/2340671/},\n\tdoi = {10.1155/2022/2340671},\n\tabstract = {Aiming at the problem of low recommendation accuracy of existing\nrecommendation algorithms, an algorithm integrating time factors and\nmultisimilarity is proposed to improve the impact of long-term data, user\nattention, and project popularity on the recommendation algorithm and the\nsimilarity of user attributes is introduced to improve the problem of cold\nstart to a certain extent. Considering that the longer the time, the less\nlikely it is to be selected again, time is introduced into the algorithm\nas a weight factor. When the behavior occurs, i.e., interest in the\nproject, so as to judge the similarity between users, not just the score\nvalue, we normalize the popularity to avoid misjudgment of high scoring\nand popular items. Because new users do not have past score records, the\nproblem of cold start can be solved by calculating the similarity of user\nattributes. Through the comparative experiment on Movielens100K dataset\nand Epinions dataset, the results show that the algorithm can improve the\naccuracy of recommendation and give users a better recommendation effect.},\n\turldate = {2022-09-05},\n\tjournal = {Math. Probl. Eng.},\n\tauthor = {Liu, Qin},\n\tmonth = aug,\n\tyear = {2022},\n\tnote = {Publisher: Hindawi},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Aiming at the problem of low recommendation accuracy of existing recommendation algorithms, an algorithm integrating time factors and multisimilarity is proposed to improve the impact of long-term data, user attention, and project popularity on the recommendation algorithm and the similarity of user attributes is introduced to improve the problem of cold start to a certain extent. Considering that the longer the time, the less likely it is to be selected again, time is introduced into the algorithm as a weight factor. When the behavior occurs, i.e., interest in the project, so as to judge the similarity between users, not just the score value, we normalize the popularity to avoid misjudgment of high scoring and popular items. Because new users do not have past score records, the problem of cold start can be solved by calculating the similarity of user attributes. Through the comparative experiment on Movielens100K dataset and Epinions dataset, the results show that the algorithm can improve the accuracy of recommendation and give users a better recommendation effect.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2207.08713\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022', 'http://arxiv.org/abs/2207.08713', event);\">\n    The Impact of Feature Quantity on Recommendation Algorithm Performance: A Movielens-100K Case Study.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wegmeth, L.\n</span>\n\n  \n\n</span>\n\n  July 2022.\n  <span class=\"note\">ISBN: 2207.08713 Publication Title: arXiv [cs.IR]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2207.08713\"\n     onclick=\"log_download('wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022', 'http://arxiv.org/abs/2207.08713', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"The Impact of Feature Quantity on Recommendation Algorithm Performance: A Movielens-100K Case Study [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>3 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wegmeth-theimpactoffeaturequantityonrecommendationalgorithmperformanceamovielens100kcasestudy-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{wegmeth_impact_2022,\n\ttitle = {The {Impact} of {Feature} {Quantity} on {Recommendation} {Algorithm} {Performance}: {A} {Movielens}-{100K} {Case} {Study}},\n\turl = {http://arxiv.org/abs/2207.08713},\n\tabstract = {Recent model-based Recommender Systems (RecSys) algorithms emphasize on\nthe use of features, also called side information, in their design similar\nto algorithms in Machine Learning (ML). In contrast, some of the most\npopular and traditional algorithms for RecSys solely focus on a given\nuser-item-rating relation without including side information. The goal of\nthis case study is to provide a performance comparison and assessment of\nRecSys and ML algorithms when side information is included. We chose the\nMovielens-100K data set since it is a standard for comparing RecSys\nalgorithms. We compared six different feature sets with varying quantities\nof features which were generated from the baseline data and evaluated on a\ntotal of 19 RecSys algorithms, baseline ML algorithms, Automated Machine\nLearning (AutoML) pipelines, and state-of-the-art RecSys algorithms that\nincorporate side information. The results show that additional features\nbenefit all algorithms we evaluated. However, the correlation between\nfeature quantity and performance is not monotonous for AutoML and RecSys.\nIn these categories, an analysis of feature importance revealed that the\nquality of features matters more than quantity. Throughout our\nexperiments, the average performance on the feature set with the lowest\nnumber of features is about 6\\% worse compared to that with the highest in\nterms of the Root Mean Squared Error. An interesting observation is that\nAutoML outperforms matrix factorization-based RecSys algorithms when\nadditional features are used. Almost all algorithms that can include side\ninformation have higher performance when using the highest quantity of\nfeatures. In the other cases, the performance difference is negligible\n({\\textless}1\\%). The results show a clear positive trend for the effect of feature\nquantity as well as the important effects of feature quality on the\nevaluated algorithms.},\n\tauthor = {Wegmeth, Lukas},\n\tmonth = jul,\n\tyear = {2022},\n\tnote = {ISBN: 2207.08713\nPublication Title: arXiv [cs.IR]},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Recent model-based Recommender Systems (RecSys) algorithms emphasize on the use of features, also called side information, in their design similar to algorithms in Machine Learning (ML). In contrast, some of the most popular and traditional algorithms for RecSys solely focus on a given user-item-rating relation without including side information. The goal of this case study is to provide a performance comparison and assessment of RecSys and ML algorithms when side information is included. We chose the Movielens-100K data set since it is a standard for comparing RecSys algorithms. We compared six different feature sets with varying quantities of features which were generated from the baseline data and evaluated on a total of 19 RecSys algorithms, baseline ML algorithms, Automated Machine Learning (AutoML) pipelines, and state-of-the-art RecSys algorithms that incorporate side information. The results show that additional features benefit all algorithms we evaluated. However, the correlation between feature quantity and performance is not monotonous for AutoML and RecSys. In these categories, an analysis of feature importance revealed that the quality of features matters more than quantity. Throughout our experiments, the average performance on the feature set with the lowest number of features is about 6% worse compared to that with the highest in terms of the Root Mean Squared Error. An interesting observation is that AutoML outperforms matrix factorization-based RecSys algorithms when additional features are used. Almost all algorithms that can include side information have higher performance when using the highest quantity of features. In the other cases, the performance difference is negligible (\\textless1%). The results show a clear positive trend for the effect of feature quantity as well as the important effects of feature quality on the evaluated algorithms.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022', 'https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf', event);\">\n    Enhancing the Multi-Objective Recommendation from three new perspectives: data characterization, risk-sensitiveness, and prioritization of the objectives.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Fortes, R. S.\n</span>\n\n  \n\n</span>\n\n  Ph.D. Thesis, Universidade Federal de Minas Gerais,  2022.\n  <span class=\"note\">Publication Title: Ciência da Computação Volume: Doctoral</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf\"\n     onclick=\"log_download('fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022', 'https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/pdf.svg\"\n\t     alt=\"Enhancing the Multi-Objective Recommendation from three new perspectives: data characterization, risk-sensitiveness, and prioritization of the objectives [pdf]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>5 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('fortes-enhancingthemultiobjectiverecommendationfromthreenewperspectivesdatacharacterizationrisksensitivenessandprioritizationoftheobjectives-2022')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@phdthesis{fortes_enhancing_2022,\n\ttitle = {Enhancing the {Multi}-{Objective} {Recommendation} from three new perspectives: data characterization, risk-sensitiveness, and prioritization of the objectives},\n\turl = {https://repositorio.ufmg.br/bitstream/1843/43915/1/Reinaldo_Silva_Fortes_Tese_UFMG.pdf},\n\tschool = {Universidade Federal de Minas Gerais},\n\tauthor = {Fortes, Renaldo Silva},\n\tcollaborator = {Gonçalves, Marcos André},\n\tyear = {2022},\n\tnote = {Publication Title: Ciência da Computação\nVolume: Doctoral},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2021\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2021')\"\n      onkeypress=\"toggleGroup('group_2021')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2021</span>\n      <span class=\"bibbase_group_count\">\n        \n        (11)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://md.ekstrandom.net/pubs/bag-extended\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'https://md.ekstrandom.net/pubs/bag-extended', event);\">\n    Exploring author gender in book rating and recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ekstrand, M. D;  and Kluver, D.\n</span>\n\n  \n\n</span>\n\n  <i>User Modeling and User-Adapted Interaction</i>, 31(3): 377–420. July 2021.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://md.ekstrandom.net/pubs/bag-extended\"\n     onclick=\"log_download('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'https://md.ekstrandom.net/pubs/bag-extended', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Exploring author gender in book rating and recommendation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s11257-020-09284-2\"\n     onclick=\"log_download('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021', 'http://doi.org/10.1007/s11257-020-09284-2', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>12 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ekstrand-kluver-exploringauthorgenderinbookratingandrecommendation-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{ekstrand_exploring_2021,\n\ttitle = {Exploring author gender in book rating and recommendation},\n\tvolume = {31},\n\tissn = {0924-1868},\n\turl = {https://md.ekstrandom.net/pubs/bag-extended},\n\tdoi = {10.1007/s11257-020-09284-2},\n\tabstract = {Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.},\n\tnumber = {3},\n\turldate = {2020-06-05},\n\tjournal = {User Modeling and User-Adapted Interaction},\n\tauthor = {Ekstrand, Michael D and Kluver, Daniel},\n\tmonth = jul,\n\tyear = {2021},\n\tpages = {377--420},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Collaborative filtering algorithms find useful patterns in rating and consumption data and exploit these patterns to guide users to good items. Many of the patterns in rating datasets reflect important real-world differences between the various users and items in the data; other patterns may be irrelevant or possibly undesirable for social or ethical reasons, particularly if they reflect undesired discrimination, such as discrimination in publishing or purchasing against authors who are women or ethnic minorities. In this work, we examine the response of collaborative filtering recommender algorithms to the distribution of their input data with respect to a dimension of social concern, namely content creator gender. Using publicly-available book ratings data, we measure the distribution of the genders of the authors of books in user rating profiles and recommendation lists produced from this data. We find that common collaborative filtering algorithms differ in the gender distribution of their recommendation lists, and in the relationship of that output distribution to user profile distribution.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021', 'https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf', event);\">\n    Exploring Set-Inspired Similarity Measures for Collaborative Filtering Recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Quang-Hung, L E;  and Thi-Xinh, L E\n</span>\n\n  \n\n</span>\n\n  In November 2021. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf\"\n     onclick=\"log_download('quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021', 'https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/pdf.svg\"\n\t     alt=\"Exploring Set-Inspired Similarity Measures for Collaborative Filtering Recommendation [pdf]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('quanghung-thixinh-exploringsetinspiredsimilaritymeasuresforcollaborativefilteringrecommendation-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{quang-hung_exploring_2021,\n\ttitle = {Exploring {Set}-{Inspired} {Similarity} {Measures} for {Collaborative} {Filtering} {Recommendation}},\n\turl = {https://iisi.siit.tu.ac.th/KSE2021/uploads_final/79__e291caf8e3a4ca8c47e9ed657a1d0f76/[KSE2021-0079]-Final-camera-ready-paper.pdf},\n\tauthor = {Quang-Hung, L E and Thi-Xinh, L E},\n\tmonth = nov,\n\tyear = {2021},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0306457321002065\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021', 'https://www.sciencedirect.com/science/article/pii/S0306457321002065', event);\">\n    Towards user-oriented privacy for recommender system data: A personalization-based approach to gender obfuscation for user profiles.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Slokom, M.; Hanjalic, A.;  and Larson, M.\n</span>\n\n  \n\n</span>\n\n  <i>Inf. Process. Manag.</i>, 58(6): 102722. November 2021.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0306457321002065\"\n     onclick=\"log_download('slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021', 'https://www.sciencedirect.com/science/article/pii/S0306457321002065', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Towards user-oriented privacy for recommender system data: A personalization-based approach to gender obfuscation for user profiles [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1016/j.ipm.2021.102722\"\n     onclick=\"log_download('slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021', 'http://doi.org/10.1016/j.ipm.2021.102722', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('slokom-hanjalic-larson-towardsuserorientedprivacyforrecommendersystemdataapersonalizationbasedapproachtogenderobfuscationforuserprofiles-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{slokom_towards_2021,\n\ttitle = {Towards user-oriented privacy for recommender system data: {A} personalization-based approach to gender obfuscation for user profiles},\n\tvolume = {58},\n\tissn = {0306-4573},\n\turl = {https://www.sciencedirect.com/science/article/pii/S0306457321002065},\n\tdoi = {10.1016/j.ipm.2021.102722},\n\tabstract = {In this paper, we propose a new privacy solution for the data used to\ntrain a recommender system, i.e., the user–item matrix. The user–item\nmatrix contains implicit information, which can be inferred using a\nclassifier, leading to potential privacy violations. Our solution, called\nPersonalized Blurring (PerBlur), is a simple, yet effective, approach to\nadding and removing items from users’ profiles in order to generate an\nobfuscated user–item matrix. The novelty of PerBlur is personalization of\nthe choice of items used for obfuscation to the individual user profiles.\nPerBlur is formulated within a user-oriented paradigm of recommender\nsystem data privacy that aims at making privacy solutions understandable,\nunobtrusive, and useful for the user. When obfuscated data is used for\ntraining, a recommender system algorithm is able to reach performance\ncomparable to what is attained when it is trained on the original,\nunobfuscated data. At the same time, a classifier can no longer reliably\nuse the obfuscated data to predict the gender of users, indicating that\nimplicit gender information has been removed. In addition to introducing\nPerBlur, we make several key contributions. First, we propose an\nevaluation protocol that creates a fair environment to compare between\ndifferent obfuscation conditions. Second, we carry out experiments that\nshow that gender obfuscation impacts the fairness and diversity of\nrecommender system results. In sum, our work establishes that a simple,\ntransparent approach to gender obfuscation can protect user privacy while\nat the same time improving recommendation results for users by maintaining\nfairness and enhancing diversity.},\n\tnumber = {6},\n\tjournal = {Inf. Process. Manag.},\n\tauthor = {Slokom, Manel and Hanjalic, Alan and Larson, Martha},\n\tmonth = nov,\n\tyear = {2021},\n\tkeywords = {Diversity, Evaluation, Fairness, Gender inference, Obfuscation, Privacy, Top-N recommendation},\n\tpages = {102722},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  In this paper, we propose a new privacy solution for the data used to train a recommender system, i.e., the user–item matrix. The user–item matrix contains implicit information, which can be inferred using a classifier, leading to potential privacy violations. Our solution, called Personalized Blurring (PerBlur), is a simple, yet effective, approach to adding and removing items from users’ profiles in order to generate an obfuscated user–item matrix. The novelty of PerBlur is personalization of the choice of items used for obfuscation to the individual user profiles. PerBlur is formulated within a user-oriented paradigm of recommender system data privacy that aims at making privacy solutions understandable, unobtrusive, and useful for the user. When obfuscated data is used for training, a recommender system algorithm is able to reach performance comparable to what is attained when it is trained on the original, unobfuscated data. At the same time, a classifier can no longer reliably use the obfuscated data to predict the gender of users, indicating that implicit gender information has been removed. In addition to introducing PerBlur, we make several key contributions. First, we propose an evaluation protocol that creates a fair environment to compare between different obfuscation conditions. Second, we carry out experiments that show that gender obfuscation impacts the fairness and diversity of recommender system results. In sum, our work establishes that a simple, transparent approach to gender obfuscation can protect user privacy while at the same time improving recommendation results for users by maintaining fairness and enhancing diversity.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2101.03054\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021', 'http://arxiv.org/abs/2101.03054', event);\">\n    Application of Knowledge Graphs to Provide Side Information for Improved Recommendation Accuracy.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Mao, Y.; Mokhov, S. A;  and Mudur, S. P\n</span>\n\n  \n\n</span>\n\n  January 2021.\n  <span class=\"note\">ISBN: 2101.03054 Publication Title: arXiv [cs.IR]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2101.03054\"\n     onclick=\"log_download('mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021', 'http://arxiv.org/abs/2101.03054', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Application of Knowledge Graphs to Provide Side Information for Improved Recommendation Accuracy [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s11257-018-9213-x\"\n     onclick=\"log_download('mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021', 'http://doi.org/10.1007/s11257-018-9213-x', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('mao-mokhov-mudur-applicationofknowledgegraphstoprovidesideinformationforimprovedrecommendationaccuracy-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{mao_application_2021,\n\ttitle = {Application of {Knowledge} {Graphs} to {Provide} {Side} {Information} for {Improved} {Recommendation} {Accuracy}},\n\turl = {http://arxiv.org/abs/2101.03054},\n\tabstract = {Personalized recommendations are popular in these days of Internet driven\nactivities, specifically shopping. Recommendation methods can be grouped\ninto three major categories, content based filtering, collaborative\nfiltering and machine learning enhanced. Information about products and\npreferences of different users are primarily used to infer preferences for\na specific user. Inadequate information can obviously cause these methods\nto fail or perform poorly. The more information we provide to these\nmethods, the more likely it is that the methods perform better. Knowledge\ngraphs represent the current trend in recording information in the form of\nrelations between entities, and can provide additional (side) information\nabout products and users. Such information can be used to improve nearest\nneighbour search, clustering users and products, or train the neural\nnetwork, when one is used. In this work, we present a new generic\nrecommendation systems framework, that integrates knowledge graphs into\nthe recommendation pipeline. We describe its software design and\nimplementation, and then show through experiments, how such a framework\ncan be specialized for a domain, say movie recommendations, and the\nimprovements in recommendation results possible due to side information\nobtained from knowledge graphs representation of such information. Our\nframework supports different knowledge graph representation formats, and\nfacilitates format conversion, merging and information extraction needed\nfor training recommendation methods.},\n\tauthor = {Mao, Yuhao and Mokhov, Serguei A and Mudur, Sudhir P},\n\tmonth = jan,\n\tyear = {2021},\n\tdoi = {10.1007/s11257-018-9213-x},\n\tnote = {ISBN: 2101.03054\nPublication Title: arXiv [cs.IR]},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Personalized recommendations are popular in these days of Internet driven activities, specifically shopping. Recommendation methods can be grouped into three major categories, content based filtering, collaborative filtering and machine learning enhanced. Information about products and preferences of different users are primarily used to infer preferences for a specific user. Inadequate information can obviously cause these methods to fail or perform poorly. The more information we provide to these methods, the more likely it is that the methods perform better. Knowledge graphs represent the current trend in recording information in the form of relations between entities, and can provide additional (side) information about products and users. Such information can be used to improve nearest neighbour search, clustering users and products, or train the neural network, when one is used. In this work, we present a new generic recommendation systems framework, that integrates knowledge graphs into the recommendation pipeline. We describe its software design and implementation, and then show through experiments, how such a framework can be specialized for a domain, say movie recommendations, and the improvements in recommendation results possible due to side information obtained from knowledge graphs representation of such information. Our framework supports different knowledge graph representation formats, and facilitates format conversion, merging and information extraction needed for training recommendation methods.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"scheidt-beel-timedependentevaluationofrecommendersystems-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://perspectives-ws.github.io/2021/proceedings/paper10.pdf\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'scheidt-beel-timedependentevaluationofrecommendersystems-2021', 'https://perspectives-ws.github.io/2021/proceedings/paper10.pdf', event);\">\n    Time-dependent evaluation of recommender systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Scheidt, T.;  and Beel, J.\n</span>\n\n  \n\n</span>\n\n  In <i>CEUR-WS</i>, volume 2955, September 2021. \n  <span class=\"note\">Journal Abbreviation: CEUR-WS</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://perspectives-ws.github.io/2021/proceedings/paper10.pdf\"\n     onclick=\"log_download('scheidt-beel-timedependentevaluationofrecommendersystems-2021', 'https://perspectives-ws.github.io/2021/proceedings/paper10.pdf', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/pdf.svg\"\n\t     alt=\"Time-dependent evaluation of recommender systems [pdf]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/scheidt-beel-timedependentevaluationofrecommendersystems-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('scheidt-beel-timedependentevaluationofrecommendersystems-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{scheidt_time-dependent_2021,\n\ttitle = {Time-dependent evaluation of recommender systems},\n\tvolume = {2955},\n\turl = {https://perspectives-ws.github.io/2021/proceedings/paper10.pdf},\n\turldate = {2021-09-27},\n\tbooktitle = {{CEUR}-{WS}},\n\tauthor = {Scheidt, Teresa and Beel, Joeran},\n\tmonth = sep,\n\tyear = {2021},\n\tnote = {Journal Abbreviation: CEUR-WS},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3472163.3472183\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021', 'https://doi.org/10.1145/3472163.3472183', event);\">\n    A Framework for Enhancing Deep Learning Based Recommender Systems with Knowledge Graphs.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Mudur, S. P; Mokhov, S. A;  and Mao, Y.\n</span>\n\n  \n\n</span>\n\n  In <i>IDEAS 2021</i>, pages 11–20, New York, NY, USA, July 2021. Association for Computing Machinery\n  <span class=\"note\">Journal Abbreviation: IDEAS 2021</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3472163.3472183\"\n     onclick=\"log_download('mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021', 'https://doi.org/10.1145/3472163.3472183', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"A Framework for Enhancing Deep Learning Based Recommender Systems with Knowledge Graphs [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3472163.3472183\"\n     onclick=\"log_download('mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021', 'http://doi.org/10.1145/3472163.3472183', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('mudur-mokhov-mao-aframeworkforenhancingdeeplearningbasedrecommendersystemswithknowledgegraphs-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{mudur_framework_2021,\n\taddress = {New York, NY, USA},\n\ttitle = {A {Framework} for {Enhancing} {Deep} {Learning} {Based} {Recommender} {Systems} with {Knowledge} {Graphs}},\n\turl = {https://doi.org/10.1145/3472163.3472183},\n\tdoi = {10.1145/3472163.3472183},\n\tabstract = {Recommendation methods fall into three major categories, content based\nfiltering, collaborative filtering and deep learning based. Information\nabout products and the preferences of earlier users are used in an\nunsupervised manner to create models which help make personalized\nrecommendations to a specific new user. The more information we provide to\nthese methods, the more likely it is that they yield better\nrecommendations. Deep learning based methods are relatively recent, and\nare generally more robust to noise and missing information. This is\nbecause deep learning models can be trained even when some of the\ninformation records have partial information. Knowledge graphs represent\nthe current trend in recording information in the form of relations\nbetween entities, and can provide any available information about products\nand users. This information is used to train the recommendation model. In\nthis work, we present a new generic recommender systems framework, that\nintegrates knowledge graphs into the recommendation pipeline. We describe\nits design and implementation, and then show through experiments, how such\na framework can be specialized, taking the domain of movies as an example,\nand the resulting improvements in recommendations made possible by using\nall the information obtained using knowledge graphs. Our framework, to be\nmade publicly available, supports different knowledge graph representation\nformats, and facilitates format conversion, merging and information\nextraction needed for training recommendation models.},\n\turldate = {2021-09-14},\n\tbooktitle = {{IDEAS} 2021},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Mudur, Sudhir P and Mokhov, Serguei A and Mao, Yuhao},\n\tmonth = jul,\n\tyear = {2021},\n\tnote = {Journal Abbreviation: IDEAS 2021},\n\tkeywords = {framework, knowledge graphs, recommendation model training, recommender system, deep learning based recommendations},\n\tpages = {11--20},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Recommendation methods fall into three major categories, content based filtering, collaborative filtering and deep learning based. Information about products and the preferences of earlier users are used in an unsupervised manner to create models which help make personalized recommendations to a specific new user. The more information we provide to these methods, the more likely it is that they yield better recommendations. Deep learning based methods are relatively recent, and are generally more robust to noise and missing information. This is because deep learning models can be trained even when some of the information records have partial information. Knowledge graphs represent the current trend in recording information in the form of relations between entities, and can provide any available information about products and users. This information is used to train the recommendation model. In this work, we present a new generic recommender systems framework, that integrates knowledge graphs into the recommendation pipeline. We describe its design and implementation, and then show through experiments, how such a framework can be specialized, taking the domain of movies as an example, and the resulting improvements in recommendations made possible by using all the information obtained using knowledge graphs. Our framework, to be made publicly available, supports different knowledge graph representation formats, and facilitates format conversion, merging and information extraction needed for training recommendation models.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2107.08959\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021', 'http://arxiv.org/abs/2107.08959', event);\">\n    T-RECS: A simulation tool to study the societal impact of recommender systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Lucherini, E.; Sun, M.; Winecoff, A.;  and Narayanan, A.\n</span>\n\n  \n\n</span>\n\n  July 2021.\n  <span class=\"note\">ISBN: 2107.08959 Publication Title: arXiv [cs.CY]</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2107.08959\"\n     onclick=\"log_download('lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021', 'http://arxiv.org/abs/2107.08959', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"T-RECS: A simulation tool to study the societal impact of recommender systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('lucherini-sun-winecoff-narayanan-trecsasimulationtooltostudythesocietalimpactofrecommendersystems-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@unpublished{lucherini_t-recs_2021,\n\ttitle = {T-{RECS}: {A} simulation tool to study the societal impact of recommender systems},\n\turl = {http://arxiv.org/abs/2107.08959},\n\tabstract = {Simulation has emerged as a popular method to study the long-term societal\nconsequences of recommender systems. This approach allows researchers to\nspecify their theoretical model explicitly and observe the evolution of\nsystem-level outcomes over time. However, performing simulation-based\nstudies often requires researchers to build their own simulation\nenvironments from the ground up, which creates a high barrier to entry,\nintroduces room for implementation error, and makes it difficult to\ndisentangle whether observed outcomes are due to the model or the\nimplementation. We introduce T-RECS, an open-sourced Python package\ndesigned for researchers to simulate recommendation systems and other\ntypes of sociotechnical systems in which an algorithm mediates the\ninteractions between multiple stakeholders, such as users and content\ncreators. To demonstrate the flexibility of T-RECS, we perform a\nreplication of two prior simulation-based research on sociotechnical\nsystems. We additionally show how T-RECS can be used to generate novel\ninsights with minimal overhead. Our tool promotes reproducibility in this\narea of research, provides a unified language for simulating\nsociotechnical systems, and removes the friction of implementing\nsimulations from scratch.},\n\tauthor = {Lucherini, Eli and Sun, Matthew and Winecoff, Amy and Narayanan, Arvind},\n\tmonth = jul,\n\tyear = {2021},\n\tnote = {ISBN: 2107.08959\nPublication Title: arXiv [cs.CY]},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Simulation has emerged as a popular method to study the long-term societal consequences of recommender systems. This approach allows researchers to specify their theoretical model explicitly and observe the evolution of system-level outcomes over time. However, performing simulation-based studies often requires researchers to build their own simulation environments from the ground up, which creates a high barrier to entry, introduces room for implementation error, and makes it difficult to disentangle whether observed outcomes are due to the model or the implementation. We introduce T-RECS, an open-sourced Python package designed for researchers to simulate recommendation systems and other types of sociotechnical systems in which an algorithm mediates the interactions between multiple stakeholders, such as users and content creators. To demonstrate the flexibility of T-RECS, we perform a replication of two prior simulation-based research on sociotechnical systems. We additionally show how T-RECS can be used to generate novel insights with minimal overhead. Our tool promotes reproducibility in this area of research, provides a unified language for simulating sociotechnical systems, and removes the friction of implementing simulations from scratch.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3404835.3462973\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021', 'https://doi.org/10.1145/3404835.3462973', event);\">\n    New Insights into Metric Optimization for Ranking-based Recommendation.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Li, R. Z.; Urbano, J.;  and Hanjalic, A.\n</span>\n\n  \n\n</span>\n\n  In <i>SIGIR '21</i>, pages 932–941, New York, NY, USA, July 2021. Association for Computing Machinery\n  <span class=\"note\">Journal Abbreviation: SIGIR '21</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3404835.3462973\"\n     onclick=\"log_download('li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021', 'https://doi.org/10.1145/3404835.3462973', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"New Insights into Metric Optimization for Ranking-based Recommendation [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3404835.3462973\"\n     onclick=\"log_download('li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021', 'http://doi.org/10.1145/3404835.3462973', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('li-urbano-hanjalic-newinsightsintometricoptimizationforrankingbasedrecommendation-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{li_new_2021,\n\taddress = {New York, NY, USA},\n\ttitle = {New {Insights} into {Metric} {Optimization} for {Ranking}-based {Recommendation}},\n\turl = {https://doi.org/10.1145/3404835.3462973},\n\tdoi = {10.1145/3404835.3462973},\n\tabstract = {Direct optimization of IR metrics has often been adopted as an approach to\ndevise and develop ranking-based recommender systems. Most methods\nfollowing this approach (e.g. TFMAP, CLiMF, Top-N-Rank) aim at optimizing\nthe same metric being used for evaluation, under the assumption that this\nwill lead to the best performance. A number of studies of this practice\nbring this assumption, however, into question. In this paper, we dig\ndeeper into this issue in order to learn more about the effects of the\nchoice of the metric to optimize on the performance of a ranking-based\nrecommender system. We present an extensive experimental study conducted\non different datasets in both pairwise and listwise learning-to-rank (LTR)\nscenarios, to compare the relative merit of four popular IR metrics,\nnamely RR, AP, nDCG and RBP, when used for optimization and assessment of\nrecommender systems in various combinations. For the first three, we\nfollow the practice of loss function formulation available in literature.\nFor the fourth one, we propose novel loss functions inspired by RBP for\nboth the pairwise and listwise scenario. Our results confirm that the best\nperformance is indeed not necessarily achieved when optimizing the same\nmetric being used for evaluation. In fact, we find that RBP-inspired\nlosses perform at least as well as other metrics in a consistent way, and\noffer clear benefits in several cases. Interesting to see is that\nRBP-inspired losses, while improving the recommendation performance for\nall uses, may lead to an individual performance gain that is correlated\nwith the activity level of a user in interacting with items. The more\nactive the users, the more they benefit. Overall, our results challenge\nthe assumption behind the current research practice of optimizing and\nevaluating the same metric, and point to RBP-based optimization instead as\na promising alternative when learning to rank in the recommendation\ncontext.},\n\turldate = {2021-07-15},\n\tbooktitle = {{SIGIR} &#x27;21},\n\tpublisher = {Association for Computing Machinery},\n\tauthor = {Li, Roger Zhe and Urbano, Julián and Hanjalic, Alan},\n\tmonth = jul,\n\tyear = {2021},\n\tnote = {Journal Abbreviation: SIGIR &#x27;21},\n\tkeywords = {evaluation metrics, learning to rank, recommender systems},\n\tpages = {932--941},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Direct optimization of IR metrics has often been adopted as an approach to devise and develop ranking-based recommender systems. Most methods following this approach (e.g. TFMAP, CLiMF, Top-N-Rank) aim at optimizing the same metric being used for evaluation, under the assumption that this will lead to the best performance. A number of studies of this practice bring this assumption, however, into question. In this paper, we dig deeper into this issue in order to learn more about the effects of the choice of the metric to optimize on the performance of a ranking-based recommender system. We present an extensive experimental study conducted on different datasets in both pairwise and listwise learning-to-rank (LTR) scenarios, to compare the relative merit of four popular IR metrics, namely RR, AP, nDCG and RBP, when used for optimization and assessment of recommender systems in various combinations. For the first three, we follow the practice of loss function formulation available in literature. For the fourth one, we propose novel loss functions inspired by RBP for both the pairwise and listwise scenario. Our results confirm that the best performance is indeed not necessarily achieved when optimizing the same metric being used for evaluation. In fact, we find that RBP-inspired losses perform at least as well as other metrics in a consistent way, and offer clear benefits in several cases. Interesting to see is that RBP-inspired losses, while improving the recommendation performance for all uses, may lead to an individual performance gain that is correlated with the activity level of a user in interacting with items. The more active the users, the more they benefit. Overall, our results challenge the assumption behind the current research practice of optimizing and evaluating the same metric, and point to RBP-based optimization instead as a promising alternative when learning to rank in the recommendation context.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0306457321001369\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021', 'https://www.sciencedirect.com/science/article/pii/S0306457321001369', event);\">\n    Fairness metrics and bias mitigation strategies for rating predictions.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ashokan, A.;  and Haas, C.\n</span>\n\n  \n\n</span>\n\n  <i>Inf. Process. Manag.</i>, 58(5): 102646. September 2021.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.sciencedirect.com/science/article/pii/S0306457321001369\"\n     onclick=\"log_download('ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021', 'https://www.sciencedirect.com/science/article/pii/S0306457321001369', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Fairness metrics and bias mitigation strategies for rating predictions [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1016/j.ipm.2021.102646\"\n     onclick=\"log_download('ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021', 'http://doi.org/10.1016/j.ipm.2021.102646', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ashokan-haas-fairnessmetricsandbiasmitigationstrategiesforratingpredictions-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{ashokan_fairness_2021,\n\ttitle = {Fairness metrics and bias mitigation strategies for rating predictions},\n\tvolume = {58},\n\tissn = {0306-4573},\n\turl = {https://www.sciencedirect.com/science/article/pii/S0306457321001369},\n\tdoi = {10.1016/j.ipm.2021.102646},\n\tabstract = {Algorithm fairness is an established line of research in the machine\nlearning domain with substantial work while the equivalent in the\nrecommender system domain is relatively new. In this article, we consider\nrating-based recommender systems which model the recommendation process as\na prediction problem. We consider different types of biases that can occur\nin this setting, discuss various fairness definitions, and also propose a\nnovel bias mitigation strategy to address potential unfairness in a\nrating-based recommender system. Based on an analysis of fairness metrics\nused in machine learning and a discussion of their applicability in the\nrecommender system domain, we map the proposed metrics from the two\ndomains and identify commonly used concepts and definitions of fairness.\nFinally, to address unfairness and potential bias against certain groups\nin a recommender system, we develop a bias mitigation algorithm and\nconduct case studies on one synthetic and one empirical dataset to show\nits effectiveness. Our results show that unfairness can be significantly\nlowered through our approach and that bias mitigation is a fruitful area\nof research for recommender systems.},\n\tnumber = {5},\n\tjournal = {Inf. Process. Manag.},\n\tauthor = {Ashokan, Ashwathy and Haas, Christian},\n\tmonth = sep,\n\tyear = {2021},\n\tkeywords = {Algorithmic fairness, Bias mitigation, Fairness metrics, Recommender systems},\n\tpages = {102646},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Algorithm fairness is an established line of research in the machine learning domain with substantial work while the equivalent in the recommender system domain is relatively new. In this article, we consider rating-based recommender systems which model the recommendation process as a prediction problem. We consider different types of biases that can occur in this setting, discuss various fairness definitions, and also propose a novel bias mitigation strategy to address potential unfairness in a rating-based recommender system. Based on an analysis of fairness metrics used in machine learning and a discussion of their applicability in the recommender system domain, we map the proposed metrics from the two domains and identify commonly used concepts and definitions of fairness. Finally, to address unfairness and potential bias against certain groups in a recommender system, we develop a bias mitigation algorithm and conduct case studies on one synthetic and one empirical dataset to show its effectiveness. Our results show that unfairness can be significantly lowered through our approach and that bias mitigation is a fruitful area of research for recommender systems.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://dx.doi.org/10.1145/3404835.3462916\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021', 'http://dx.doi.org/10.1145/3404835.3462916', event);\">\n    Standing in Your Shoes: External Assessments for Personalized Recommender Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Lu, H.; Ma, W.; Zhang, M.; de Rijke, M.; Liu, Y.;  and Ma, S.\n</span>\n\n  \n\n</span>\n\n  In  2021. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://dx.doi.org/10.1145/3404835.3462916\"\n     onclick=\"log_download('lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021', 'http://dx.doi.org/10.1145/3404835.3462916', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Standing in Your Shoes: External Assessments for Personalized Recommender Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3404835.3462916\"\n     onclick=\"log_download('lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021', 'http://doi.org/10.1145/3404835.3462916', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>21 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('lu-ma-zhang-derijke-liu-ma-standinginyourshoesexternalassessmentsforpersonalizedrecommendersystems-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{lu_standing_2021,\n\ttitle = {Standing in {Your} {Shoes}: {External} {Assessments} for {Personalized} {Recommender} {Systems}},\n\turl = {http://dx.doi.org/10.1145/3404835.3462916},\n\tdoi = {10.1145/3404835.3462916},\n\tauthor = {Lu, Hongyu and Ma, Weizhi and Zhang, Min and de Rijke, Maarten and Liu, Yiqun and Ma, Shaoping},\n\tyear = {2021},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://repository.unad.edu.co/handle/10596/40465\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021', 'https://repository.unad.edu.co/handle/10596/40465', event);\">\n    Sistema de recomendación de matrícula de cursos electivos para estudiantes de Ingeniería Electrónica e Ingeniería de Telecomunicaciones de la UNAD.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Noguera Torres, A. d. P.\n</span>\n\n  \n\n</span>\n\n  Master's thesis, Universidad Nacional Abierta y a Distancia, April 2021.\n  <span class=\"note\">Volume: MS in Information Technology</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://repository.unad.edu.co/handle/10596/40465\"\n     onclick=\"log_download('nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021', 'https://repository.unad.edu.co/handle/10596/40465', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Sistema de recomendación de matrícula de cursos electivos para estudiantes de Ingeniería Electrónica e Ingeniería de Telecomunicaciones de la UNAD [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('nogueratorres-sistemaderecomendacindematrculadecursoselectivosparaestudiantesdeingenieraelectrnicaeingenieradetelecomunicacionesdelaunad-2021')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@mastersthesis{noguera_torres_sistema_2021,\n\ttitle = {Sistema de recomendación de matrícula de cursos electivos para estudiantes de {Ingeniería} {Electrónica} e {Ingeniería} de {Telecomunicaciones} de la {UNAD}},\n\turl = {https://repository.unad.edu.co/handle/10596/40465},\n\turldate = {2021-05-12},\n\tschool = {Universidad Nacional Abierta y a Distancia},\n\tauthor = {Noguera Torres, Adriana del Pilar},\n\tcollaborator = {Rúa Pérez, Santiago},\n\tmonth = apr,\n\tyear = {2021},\n\tnote = {Volume: MS in Information Technology},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2020\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2020')\"\n      onkeypress=\"toggleGroup('group_2020')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2020</span>\n      <span class=\"bibbase_group_count\">\n        \n        (8)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2004.13157\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020', 'http://arxiv.org/abs/2004.13157', event);\">\n    Evaluating Stochastic Rankings with Expected Exposure.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Diaz, F.; Mitra, B.; Ekstrand, M. D; Biega, A. J;  and Carterette, B.\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 29th ACM International Conference on Information and Knowledge Management</i>, of <i>CIKM '20</i>, October 2020. ACM\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2004.13157\"\n     onclick=\"log_download('diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020', 'http://arxiv.org/abs/2004.13157', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Evaluating Stochastic Rankings with Expected Exposure [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3340531.3411962\"\n     onclick=\"log_download('diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020', 'http://doi.org/10.1145/3340531.3411962', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('diaz-mitra-ekstrand-biega-carterette-evaluatingstochasticrankingswithexpectedexposure-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{diaz_evaluating_2020,\n\tseries = {{CIKM} &#x27;20},\n\ttitle = {Evaluating {Stochastic} {Rankings} with {Expected} {Exposure}},\n\turl = {http://arxiv.org/abs/2004.13157},\n\tdoi = {10.1145/3340531.3411962},\n\tabstract = {We introduce the concept of expected exposure as the average attention\nranked items receive from users over repeated samples of the same query.\nFurthermore, we advocate for the adoption of the principle of equal\nexpected exposure: given a fixed information need, no item receive more or\nless expected exposure compared to any other item of the same relevance\ngrade. We argue that this principle is desirable for many retrieval\nobjectives and scenarios, including topical diversity and fair ranking.\nLeveraging user models from existing retrieval metrics, we propose a\ngeneral evaluation methodology based on expected exposure and draw\nconnections to related metrics in information retrieval evaluation.\nImportantly, this methodology relaxes classic information retrieval\nassumptions, allowing a system, in response to a query, to produce a\ndistribution over rankings instead of a single fixed ranking. We study the\nbehavior of the expected exposure metric and stochastic rankers across a\nvariety of information access conditions, including ad hoc retrieval and\nrecommendation. We believe that measuring and optimizing expected exposure\nmetrics using randomization opens a new area for retrieval algorithm\ndevelopment and progress.},\n\tbooktitle = {Proceedings of the 29th {ACM} {International} {Conference} on {Information} and {Knowledge} {Management}},\n\tpublisher = {ACM},\n\tauthor = {Diaz, Fernando and Mitra, Bhaskar and Ekstrand, Michael D and Biega, Asia J and Carterette, Ben},\n\tmonth = oct,\n\tyear = {2020},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  We introduce the concept of expected exposure as the average attention ranked items receive from users over repeated samples of the same query. Furthermore, we advocate for the adoption of the principle of equal expected exposure: given a fixed information need, no item receive more or less expected exposure compared to any other item of the same relevance grade. We argue that this principle is desirable for many retrieval objectives and scenarios, including topical diversity and fair ranking. Leveraging user models from existing retrieval metrics, we propose a general evaluation methodology based on expected exposure and draw connections to related metrics in information retrieval evaluation. Importantly, this methodology relaxes classic information retrieval assumptions, allowing a system, in response to a query, to produce a distribution over rankings instead of a single fixed ranking. We study the behavior of the expected exposure metric and stochastic rankers across a variety of information access conditions, including ad hoc retrieval and recommendation. We believe that measuring and optimizing expected exposure metrics using randomization opens a new area for retrieval algorithm development and progress.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://arxiv.org/abs/2009.01311\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020', 'http://arxiv.org/abs/2009.01311', event);\">\n    Comparing fair ranking metrics.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Raj, A.; Wood, C.; Montoly, A.;  and Ekstrand, M. D\n</span>\n\n  \n\n</span>\n\n  In September 2020. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://arxiv.org/abs/2009.01311\"\n     onclick=\"log_download('raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020', 'http://arxiv.org/abs/2009.01311', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Comparing fair ranking metrics [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('raj-wood-montoly-ekstrand-comparingfairrankingmetrics-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{raj_comparing_2020,\n\ttitle = {Comparing fair ranking metrics},\n\turl = {http://arxiv.org/abs/2009.01311},\n\tabstract = {Ranking is a fundamental aspect of recommender systems. However, ranked\noutputs can be susceptible to various biases; some of these may cause\ndisadvantages to members of protected groups. Several metrics have been\nproposed to quantify the (un)fairness of rankings, but there has not been\nto date any direct comparison of these metrics. This complicates deciding\nwhat fairness metrics are applicable for specific scenarios, and assessing\nthe extent to which metrics agree or disagree. In this paper, we describe\nseveral fair ranking metrics in a common notation, enabling direct\ncomparison of their approaches and assumptions, and empirically compare\nthem on the same experimental setup and data set. Our work provides a\ndirect comparative analysis identifying similarities and differences of\nfair ranking metrics selected for our work.},\n\tauthor = {Raj, Amifa and Wood, Connor and Montoly, Ananda and Ekstrand, Michael D},\n\tmonth = sep,\n\tyear = {2020},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Ranking is a fundamental aspect of recommender systems. However, ranked outputs can be susceptible to various biases; some of these may cause disadvantages to members of protected groups. Several metrics have been proposed to quantify the (un)fairness of rankings, but there has not been to date any direct comparison of these metrics. This complicates deciding what fairness metrics are applicable for specific scenarios, and assessing the extent to which metrics agree or disagree. In this paper, we describe several fair ranking metrics in a common notation, enabling direct comparison of their approaches and assumptions, and empirically compare them on the same experimental setup and data set. Our work provides a direct comparative analysis identifying similarities and differences of fair ranking metrics selected for our work.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1145/3343413.3378004\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020', 'https://doi.org/10.1145/3343413.3378004', event);\">\n    Estimating Error and Bias in Offline Evaluation Results.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Tian, M.;  and Ekstrand, M. D\n</span>\n\n  \n\n</span>\n\n  In <i>Proceedings of the 2020 Conference on Human Information Interaction and Retrieval</i>, of <i>CHIIR '20</i>, pages 392–396, March 2020. ACM\n  <span class=\"note\">Journal Abbreviation: CHIIR '20</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1145/3343413.3378004\"\n     onclick=\"log_download('tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020', 'https://doi.org/10.1145/3343413.3378004', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Estimating Error and Bias in Offline Evaluation Results [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1145/3343413.3378004\"\n     onclick=\"log_download('tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020', 'http://doi.org/10.1145/3343413.3378004', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>4 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('tian-ekstrand-estimatingerrorandbiasinofflineevaluationresults-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{tian_estimating_2020,\n\tseries = {{CHIIR} &#x27;20},\n\ttitle = {Estimating {Error} and {Bias} in {Offline} {Evaluation} {Results}},\n\turl = {https://doi.org/10.1145/3343413.3378004},\n\tdoi = {10.1145/3343413.3378004},\n\turldate = {2020-03-26},\n\tbooktitle = {Proceedings of the 2020 {Conference} on {Human} {Information} {Interaction} and {Retrieval}},\n\tpublisher = {ACM},\n\tauthor = {Tian, Mucun and Ekstrand, Michael D},\n\tmonth = mar,\n\tyear = {2020},\n\tnote = {Journal Abbreviation: CHIIR &#x27;20},\n\tkeywords = {offline evaluation, simulation},\n\tpages = {392--396},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020', 'https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/', event);\">\n    Impact of End-User Privacy Enhancing Technologies (PETs) on Firms’ Analytics Performance.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Chen, D.;  and Hahn, J.\n</span>\n\n  \n\n</span>\n\n  In  2020. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/\"\n     onclick=\"log_download('chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020', 'https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Impact of End-User Privacy Enhancing Technologies (PETs) on Firms’ Analytics Performance [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('chen-hahn-impactofenduserprivacyenhancingtechnologiespetsonfirmsanalyticsperformance-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{chen_impact_2020,\n\ttitle = {Impact of {End}-{User} {Privacy} {Enhancing} {Technologies} ({PETs}) on {Firms}’ {Analytics} {Performance}},\n\turl = {https://aisel.aisnet.org/icis2020/digital_commerce/digital_commerce/3/},\n\tabstract = {Big data analytics in digital commerce requires vast amounts of personal\ninformation from consumers, but this gives rise to major privacy concerns.\nTo combat the threat of privacy invasion, individuals are proactively\nadopting privacy enhancing technologies (PETs) to protect their personal\ninformation. Consumers’ adoption of PETs may hamper firms’ big data\nanalytics capabilities and performance but our knowledge of how PETs\nimpact firms’ data analytics is limited. This study proposes an\ninductively derived framework which qualitatively shows that end-user PETs\ninduce measurement error and/or missing values with regards to attributes,\nentities and relationships in firms’ customer databases, but the impacts\nof specific end-user PETs may vary by analytics use case. Our simulation\nexperiments in the context of product recommendations quantitively find\nthat consumers’ adoption characteristics (adoption rate and pattern) and\nPETs characteristics (protection mechanism and intensity) significantly\naffect the performance of recommender systems.},\n\turldate = {2021-01-28},\n\tauthor = {Chen, Dawei and Hahn, Jungpil},\n\tyear = {2020},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Big data analytics in digital commerce requires vast amounts of personal information from consumers, but this gives rise to major privacy concerns. To combat the threat of privacy invasion, individuals are proactively adopting privacy enhancing technologies (PETs) to protect their personal information. Consumers’ adoption of PETs may hamper firms’ big data analytics capabilities and performance but our knowledge of how PETs impact firms’ data analytics is limited. This study proposes an inductively derived framework which qualitatively shows that end-user PETs induce measurement error and/or missing values with regards to attributes, entities and relationships in firms’ customer databases, but the impacts of specific end-user PETs may vary by analytics use case. Our simulation experiments in the context of product recommendations quantitively find that consumers’ adoption characteristics (adoption rate and pattern) and PETs characteristics (protection mechanism and intensity) significantly affect the performance of recommender systems.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://dx.doi.org/10.1109/BMSB49480.2020.9379914\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020', 'http://dx.doi.org/10.1109/BMSB49480.2020.9379914', event);\">\n    A Recommender Systems’ algorithm evaluation using the Lenskit library and MovieLens databases.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Paullier, A.;  and Sotelo, R.\n</span>\n\n  \n\n</span>\n\n  In pages 1–7, October 2020. \n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://dx.doi.org/10.1109/BMSB49480.2020.9379914\"\n     onclick=\"log_download('paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020', 'http://dx.doi.org/10.1109/BMSB49480.2020.9379914', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"A Recommender Systems’ algorithm evaluation using the Lenskit library and MovieLens databases [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1109/BMSB49480.2020.9379914\"\n     onclick=\"log_download('paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020', 'http://doi.org/10.1109/BMSB49480.2020.9379914', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('paullier-sotelo-arecommendersystemsalgorithmevaluationusingthelenskitlibraryandmovielensdatabases-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{paullier_recommender_2020,\n\ttitle = {A {Recommender} {Systems}’ algorithm evaluation using the {Lenskit} library and {MovieLens} databases},\n\turl = {http://dx.doi.org/10.1109/BMSB49480.2020.9379914},\n\tdoi = {10.1109/BMSB49480.2020.9379914},\n\tabstract = {A wide variety of algorithms can be found in the recommender system&#x27;s\nliterature for computing predictions and obtaining recommendation lists.\nThese algorithms&#x27; performance can be evaluated by numerous metrics\ndepending on which business goal is intended to be optimized. However,\nperformance can vary drastically depending on the selected algorithm as\nwell as the selected metric. In addition, algorithmic performance can be\nheavily influenced by the experimental setup such as: dataset,\ncross-validation strategy, recommender framework and the model\nhyper-parameters chosen. In this work we will compare nine different\nalgorithms used for generating recommendations across seven different\nmetrics. One vital aspect of our analysis is to provide a detailed and\nclear explanation of the methodology carried out as well as the\nexperimental setup in order to ensure future reproducibility. Moreover,\nperformance and computation times will be introduced for further\ncomparison. Finally, optimal model hyper-parameter combinations will be\npresented for benchmarking purposes.},\n\tauthor = {Paullier, Alejo and Sotelo, Rafael},\n\tmonth = oct,\n\tyear = {2020},\n\tkeywords = {Algorithms, Business, Computational modeling, Databases, Evaluation, LensKit, Libraries, MovieLens, Multimedia communication, Prediction algorithms, Recommender systems, Reproducibility of results},\n\tpages = {1--7},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  A wide variety of algorithms can be found in the recommender system's literature for computing predictions and obtaining recommendation lists. These algorithms' performance can be evaluated by numerous metrics depending on which business goal is intended to be optimized. However, performance can vary drastically depending on the selected algorithm as well as the selected metric. In addition, algorithmic performance can be heavily influenced by the experimental setup such as: dataset, cross-validation strategy, recommender framework and the model hyper-parameters chosen. In this work we will compare nine different algorithms used for generating recommendations across seven different metrics. One vital aspect of our analysis is to provide a detailed and clear explanation of the methodology carried out as well as the experimental setup in order to ensure future reproducibility. Moreover, performance and computation times will be introduced for further comparison. Finally, optimal model hyper-parameter combinations will be presented for benchmarking purposes.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ziegler-loepp-empfehlungssysteme-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1007/978-3-658-17291-6_52\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ziegler-loepp-empfehlungssysteme-2020', 'https://doi.org/10.1007/978-3-658-17291-6_52', event);\">\n    Empfehlungssysteme.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ziegler, J.;  and Loepp, B.\n</span>\n\n  \n\n</span>\n\n  In Kollmann, T., editor(s), <i>Handbuch Digitale Wirtschaft</i>, pages 717–741. Springer Fachmedien Wiesbaden, Wiesbaden,  2020.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1007/978-3-658-17291-6_52\"\n     onclick=\"log_download('ziegler-loepp-empfehlungssysteme-2020', 'https://doi.org/10.1007/978-3-658-17291-6_52', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Empfehlungssysteme [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-3-658-17291-6_52\"\n     onclick=\"log_download('ziegler-loepp-empfehlungssysteme-2020', 'http://doi.org/10.1007/978-3-658-17291-6_52', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ziegler-loepp-empfehlungssysteme-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>1 download</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ziegler-loepp-empfehlungssysteme-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@incollection{ziegler_empfehlungssysteme_2020,\n\taddress = {Wiesbaden},\n\ttitle = {Empfehlungssysteme},\n\tisbn = {978-3-658-17291-6},\n\turl = {https://doi.org/10.1007/978-3-658-17291-6_52},\n\tabstract = {Empfehlungssysteme stellen heute eine zentrale Komponente vieler\nOnline-Plattformen dar, die bei Online-Shops und vielen anderen\nAnwendungen häufig zum Einsatz kommt. Ziel ist es, dem Kunden entsprechend\nseinen persönlichen Präferenzen Produkte oder andere Artikel\nvorzuschlagen, die für ihn von Interesse sind und potenziell zu einem Kauf\noder generell zur Nutzung führen. Empfehlungssysteme haben eine erhebliche\nwirtschaftliche Bedeutung, da sie in vielen Fällen zu einem signifikanten\nAnteil zu Erfolgsfaktoren wie Click-through-Raten oder Käufen beitragen.\nWir stellen in diesem Kapitel die unterschiedlichen Ansätze zur\nautomatisierten Empfehlungsgebung vor und beschreiben konkrete Techniken\nzu deren Umsetzung. Weiterhin gehen wir auf wesentliche Aspekte der\nGestaltung und Bewertung von Empfehlungssystemen ein und diskutieren\nanwenderrelevante Themen wie Usability und Vertrauen in systemgenerierte\nEmpfehlungen.},\n\tbooktitle = {Handbuch {Digitale} {Wirtschaft}},\n\tpublisher = {Springer Fachmedien Wiesbaden},\n\tauthor = {Ziegler, Jürgen and Loepp, Benedikt},\n\teditor = {Kollmann, Tobias},\n\tyear = {2020},\n\tdoi = {10.1007/978-3-658-17291-6_52},\n\tpages = {717--741},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Empfehlungssysteme stellen heute eine zentrale Komponente vieler Online-Plattformen dar, die bei Online-Shops und vielen anderen Anwendungen häufig zum Einsatz kommt. Ziel ist es, dem Kunden entsprechend seinen persönlichen Präferenzen Produkte oder andere Artikel vorzuschlagen, die für ihn von Interesse sind und potenziell zu einem Kauf oder generell zur Nutzung führen. Empfehlungssysteme haben eine erhebliche wirtschaftliche Bedeutung, da sie in vielen Fällen zu einem signifikanten Anteil zu Erfolgsfaktoren wie Click-through-Raten oder Käufen beitragen. Wir stellen in diesem Kapitel die unterschiedlichen Ansätze zur automatisierten Empfehlungsgebung vor und beschreiben konkrete Techniken zu deren Umsetzung. Weiterhin gehen wir auf wesentliche Aspekte der Gestaltung und Bewertung von Empfehlungssystemen ein und diskutieren anwenderrelevante Themen wie Usability und Vertrauen in systemgenerierte Empfehlungen.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://doi.org/10.1007/s11042-020-09061-8\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020', 'https://doi.org/10.1007/s11042-020-09061-8', event);\">\n    Evaluating facial recognition services as interaction technique for recommender systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  De Pessemier, T.; Coppens, I.;  and Martens, L.\n</span>\n\n  \n\n</span>\n\n  <i>Multimed. Tools Appl.</i>. June 2020.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://doi.org/10.1007/s11042-020-09061-8\"\n     onclick=\"log_download('depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020', 'https://doi.org/10.1007/s11042-020-09061-8', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Evaluating facial recognition services as interaction technique for recommender systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/s11042-020-09061-8\"\n     onclick=\"log_download('depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020', 'http://doi.org/10.1007/s11042-020-09061-8', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>5 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('depessemier-coppens-martens-evaluatingfacialrecognitionservicesasinteractiontechniqueforrecommendersystems-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{de_pessemier_evaluating_2020,\n\ttitle = {Evaluating facial recognition services as interaction technique for recommender systems},\n\tissn = {1380-7501},\n\turl = {https://doi.org/10.1007/s11042-020-09061-8},\n\tdoi = {10.1007/s11042-020-09061-8},\n\tabstract = {Recommender systems are tools and techniques to assist users in the\ncontent selection process thereby coping with the problem of information\noverload. For recommender systems, user authentication and feedback\ngathering are of crucial importance. However, the typical user\nauthentication with username / password and feedback method with a star\nrating system are not user friendly and often bypassed. This article\nproposes an alternative method for user authentication based on facial\nrecognition and an automatic feedback gathering method by detecting\nvarious face characteristics such as emotions. We studied the use case of\nvideo watching. Photos made with the front-facing camera of a tablet,\nsmartphone, or smart TV are used as input of a facial recognition service.\nThe persons in front of the screen can be identified. During video\nwatching, implicit feedback for the video content is automatically\ngathered through emotion recognition, attention measurements, and behavior\nanalysis. An evaluation with a test panel showed that the recognized\nemotions are correlated with the user’s star ratings and that happiness\ncan be most accurately detected. So as the main contribution, this article\nindicates that emotion recognition might be used as an alternative\nfeedback mechanism for recommender systems.},\n\tjournal = {Multimed. Tools Appl.},\n\tauthor = {De Pessemier, Toon and Coppens, Ine and Martens, Luc},\n\tmonth = jun,\n\tyear = {2020},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Recommender systems are tools and techniques to assist users in the content selection process thereby coping with the problem of information overload. For recommender systems, user authentication and feedback gathering are of crucial importance. However, the typical user authentication with username / password and feedback method with a star rating system are not user friendly and often bypassed. This article proposes an alternative method for user authentication based on facial recognition and an automatic feedback gathering method by detecting various face characteristics such as emotions. We studied the use case of video watching. Photos made with the front-facing camera of a tablet, smartphone, or smart TV are used as input of a facial recognition service. The persons in front of the screen can be identified. During video watching, implicit feedback for the video content is automatically gathered through emotion recognition, attention measurements, and behavior analysis. An evaluation with a test panel showed that the recognized emotions are correlated with the user’s star ratings and that happiness can be most accurately detected. So as the main contribution, this article indicates that emotion recognition might be used as an alternative feedback mechanism for recommender systems.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020', 'https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738', event);\">\n    CBRec: a book recommendation system for children using the matrix factorisation and content-based filtering approaches.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Ng, Y.\n</span>\n\n  \n\n</span>\n\n  <i>International Journal of Business Intelligence and Data Mining</i>, 16(2): 129–149. January 2020.\n  <span class=\"note\">Publisher: Inderscience Publishers</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738\"\n     onclick=\"log_download('ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020', 'https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"CBRec: a book recommendation system for children using the matrix factorisation and content-based filtering approaches [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1504/IJBIDM.2020.104738\"\n     onclick=\"log_download('ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020', 'http://doi.org/10.1504/IJBIDM.2020.104738', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>9 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('ng-cbrecabookrecommendationsystemforchildrenusingthematrixfactorisationandcontentbasedfilteringapproaches-2020')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@article{ng_cbrec_2020,\n\ttitle = {{CBRec}: a book recommendation system for children using the matrix factorisation and content-based filtering approaches},\n\tvolume = {16},\n\tissn = {1743-8187},\n\turl = {https://www.inderscienceonline.com/doi/abs/10.1504/IJBIDM.2020.104738},\n\tdoi = {10.1504/IJBIDM.2020.104738},\n\tabstract = {Promoting good reading habits among children is essential, given the\nenormous influence of reading on students&#x27; development as learners and\nmembers of the society. Unfortunately, very few (children) websites or\nonline applications recommend books to children, even though they can play\na significant role in encouraging children to read. Given that a few\npopular book websites suggest books to children based on the popularity of\nbooks or rankings on books, they are not customised/personalised for each\nindividual user and likely recommend books that users do not want or like.\nWe have integrated the matrix factorisation approach and the content-based\napproach, in addition to predicting the grade levels of books, to\nrecommend books for children. Recent research works have demonstrated that\na hybrid approach, which combines different filtering approaches, is more\neffective in making recommendations. Conducted empirical study has\nverified the effectiveness of our proposed children book recommendation\nsystem.},\n\tnumber = {2},\n\tjournal = {International Journal of Business Intelligence and Data Mining},\n\tauthor = {Ng, Yiu-Kai},\n\tmonth = jan,\n\tyear = {2020},\n\tnote = {Publisher: Inderscience Publishers},\n\tpages = {129--149},\n}\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Promoting good reading habits among children is essential, given the enormous influence of reading on students' development as learners and members of the society. Unfortunately, very few (children) websites or online applications recommend books to children, even though they can play a significant role in encouraging children to read. Given that a few popular book websites suggest books to children based on the popularity of books or rankings on books, they are not customised/personalised for each individual user and likely recommend books that users do not want or like. We have integrated the matrix factorisation approach and the content-based approach, in addition to predicting the grade levels of books, to recommend books for children. Recent research works have demonstrated that a hybrid approach, which combines different filtering approaches, is more effective in making recommendations. Conducted empirical study has verified the effectiveness of our proposed children book recommendation system.\n</div>\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n  <div class=\"bibbase_group_whole\" id=\"group_2019\">\n    <div class=\"bibbase_group\"\n      onclick=\"toggleGroup('group_2019')\"\n      onkeypress=\"toggleGroup('group_2019')\"\n      tabindex=\"0\">\n      <i class=\"fa fa-angle-down bibbase_group_head_icon\"></i>&nbsp;\n      <span>2019</span>\n      <span class=\"bibbase_group_count\">\n        \n        (5)\n        \n      </span>\n    </div>\n    <div class=\"bibbase_group_body\">\n      \n  \n  <div class=\"bibbase_paper\" id=\"narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"https://conservancy.umn.edu/handle/11299/206148\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019', 'https://conservancy.umn.edu/handle/11299/206148', event);\">\n    What is the Value of Rating Obscure Items? An Analysis of the Effect of Less-Popular Items on Recommendation Quality.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Narayan, A\n</span>\n\n  \n\n</span>\n\n  Master's thesis, University of Minnesota,  2019.\n  <span class=\"note\">Publication Title: Department of Computer Science Volume: M.S. in Computer Science</span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"https://conservancy.umn.edu/handle/11299/206148\"\n     onclick=\"log_download('narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019', 'https://conservancy.umn.edu/handle/11299/206148', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"What is the Value of Rating Obscure Items? An Analysis of the Effect of Less-Popular Items on Recommendation Quality [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>2 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('narayan-whatisthevalueofratingobscureitemsananalysisoftheeffectoflesspopularitemsonrecommendationquality-2019')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@mastersthesis{narayan_what_2019,\n\ttitle = {What is the {Value} of {Rating} {Obscure} {Items}? {An} {Analysis} of the {Effect} of {Less}-{Popular} {Items} on {Recommendation} {Quality}},\n\turl = {https://conservancy.umn.edu/handle/11299/206148},\n\tabstract = {Recommender systems designers believe that the system stands to benefit\nfrom the users rating items that do not have many ratings. However, the\neffect of this act of rating lesser known items on the user&#x27;s\nrecommendations is unknown. This leads to asking the question of whether\nthese low popularity items affect the recommendations received by users.\nThis work looks at the effect less popular items have on a user&#x27;s\nrecommendations and the prediction and recommendations metrics that\nquantify the quality of recommendations …},\n\tschool = {University of Minnesota},\n\tauthor = {Narayan, A},\n\tyear = {2019},\n\tnote = {Publication Title: Department of Computer Science\nVolume: M.S. in Computer Science},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Recommender systems designers believe that the system stands to benefit from the users rating items that do not have many ratings. However, the effect of this act of rating lesser known items on the user's recommendations is unknown. This leads to asking the question of whether these low popularity items affect the recommendations received by users. This work looks at the effect less popular items have on a user's recommendations and the prediction and recommendations metrics that quantify the quality of recommendations …\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://ceur-ws.org/Vol-2450/paper4.pdf\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019', 'http://ceur-ws.org/Vol-2450/paper4.pdf', event);\">\n    Using facial recognition services as implicit feedback for recommenders.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Pessemier, T. D.; Coppens, I.;  and Martens, L.\n</span>\n\n  \n\n</span>\n\n  In volume 2450, pages 8,  2019. CEUR-WS\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://ceur-ws.org/Vol-2450/paper4.pdf\"\n     onclick=\"log_download('pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019', 'http://ceur-ws.org/Vol-2450/paper4.pdf', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/pdf.svg\"\n\t     alt=\"Using facial recognition services as implicit feedback for recommenders [pdf]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>3 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('pessemier-coppens-martens-usingfacialrecognitionservicesasimplicitfeedbackforrecommenders-2019')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{pessemier_using_2019,\n\ttitle = {Using facial recognition services as implicit feedback for recommenders},\n\tvolume = {2450},\n\turl = {http://ceur-ws.org/Vol-2450/paper4.pdf},\n\tabstract = {User authentication and feedback gathering are crucial aspects for\nrecommender systems. The most common implementations, a username /\npassword login and star rating systems, require user interaction and a\ncognitive effort from the user. As a result, users opt to save their\npassword in the interface and optional feedback with a star rating system\nis often skipped, especially for applications such as video watching in a\nhome environment. In this article, we propose an alternative method for\nuser authentication based on facial recognition and an automatic feedback\ngathering method by detecting various face characteristics. Using facial\nrecognition with a camera in a tablet, smartphone, or smart TV, the\npersons in front of the screen can be identified in order to link video\nwatching sessions to their user profile. During video watching, implicit\nfeedback is automatically gathered through emotion recognition, attention\nmeasurements, and behavior analysis. An emotion fingerprint, which is\ndefined as a unique spectrum of expected emotions for a video scene, is\ncompared to the recognized emotions in order to estimate the experience of\na user while watching. An evaluation with a test panel showed that\nhappiness can be most accurately detected and the recognized emotions are\ncorrelated with the user’s star rating.},\n\tpublisher = {CEUR-WS},\n\tauthor = {Pessemier, Toon De and Coppens, Ine and Martens, Luc},\n\tyear = {2019},\n\tpages = {8},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  User authentication and feedback gathering are crucial aspects for recommender systems. The most common implementations, a username / password login and star rating systems, require user interaction and a cognitive effort from the user. As a result, users opt to save their password in the interface and optional feedback with a star rating system is often skipped, especially for applications such as video watching in a home environment. In this article, we propose an alternative method for user authentication based on facial recognition and an automatic feedback gathering method by detecting various face characteristics. Using facial recognition with a camera in a tablet, smartphone, or smart TV, the persons in front of the screen can be identified in order to link video watching sessions to their user profile. During video watching, implicit feedback is automatically gathered through emotion recognition, attention measurements, and behavior analysis. An emotion fingerprint, which is defined as a unique spectrum of expected emotions for a video scene, is compared to the recognized emotions in order to estimate the experience of a user while watching. An evaluation with a test panel showed that happiness can be most accurately detected and the recognized emotions are correlated with the user’s star rating.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://dx.doi.org/10.1109/ICTAI.2019.00253\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019', 'http://dx.doi.org/10.1109/ICTAI.2019.00253', event);\">\n    Bayesian Deep Learning Based Exploration-Exploitation for Personalized Recommendations.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Wang, X;  and Kadioglu, S\n</span>\n\n  \n\n</span>\n\n  In pages 1715–1719, November 2019. ieeexplore.ieee.org\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://dx.doi.org/10.1109/ICTAI.2019.00253\"\n     onclick=\"log_download('wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019', 'http://dx.doi.org/10.1109/ICTAI.2019.00253', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Bayesian Deep Learning Based Exploration-Exploitation for Personalized Recommendations [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1109/ICTAI.2019.00253\"\n     onclick=\"log_download('wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019', 'http://doi.org/10.1109/ICTAI.2019.00253', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n  &nbsp;\n  <a class=\"bibbase_abstract_link bibbase link\"\n     href=\"#\"\n     onclick=\"showAbstract(event); return false;\">\n    abstract <i class=\"fa fa-caret-down\"></i></a>\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>4 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('wang-kadioglu-bayesiandeeplearningbasedexplorationexploitationforpersonalizedrecommendations-2019')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@inproceedings{wang_bayesian_2019,\n\ttitle = {Bayesian {Deep} {Learning} {Based} {Exploration}-{Exploitation} for {Personalized} {Recommendations}},\n\turl = {http://dx.doi.org/10.1109/ICTAI.2019.00253},\n\tdoi = {10.1109/ICTAI.2019.00253},\n\tabstract = {Personalized Recommendation Systems require an effective method to balance\nexploration and exploitation. To learn effective strategies, user and item\nattributes are critical data sources to capture contextual information. In\nthis paper, we first present an approach based on Bayesian Deep Learning\nto learn a compact representation of user and item attributes to guide\nexploitation. A key novelty of the approach lies in its ability to also\ncapture the uncertainty associated with the model output to guide\nexploration. We then show how to further boost exploration by\nincorporating model uncertainty with that of data uncertainty.\nExperimental results demonstrate the benefits of our approach in terms of\naccuracy in recommendations as well as its performance in an online\nsetting.},\n\tpublisher = {ieeexplore.ieee.org},\n\tauthor = {Wang, X and Kadioglu, S},\n\tmonth = nov,\n\tyear = {2019},\n\tkeywords = {Bayesian Deep Learning, ExplorationExploitation, Personalized Recommendation, Bayesian deep learning, boost exploration, compact representation, contextual information, critical data sources, data uncertainty, exploration-exploitation, information filtering, item attributes, learning (artificial intelligence), model output, model uncertainty, personalized recommendation systems, recommender systems},\n\tpages = {1715--1719},\n}\n\n</pre>\n</div>\n\n\n<div class=\"well well-small bibbase\" data-type=\"abstract\"\n     style=\"display:none\">\n  Personalized Recommendation Systems require an effective method to balance exploration and exploitation. To learn effective strategies, user and item attributes are critical data sources to capture contextual information. In this paper, we first present an approach based on Bayesian Deep Learning to learn a compact representation of user and item attributes to guide exploitation. A key novelty of the approach lies in its ability to also capture the uncertainty associated with the model output to guide exploration. We then show how to further boost exploration by incorporating model uncertainty with that of data uncertainty. Experimental results demonstrate the benefits of our approach in terms of accuracy in recommendations as well as its performance in an online setting.\n</div>\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"varga-recommendersystems-2019\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://dx.doi.org/10.1007/978-1-4842-4859-1\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'varga-recommendersystems-2019', 'http://dx.doi.org/10.1007/978-1-4842-4859-1', event);\">\n    Recommender Systems.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Varga, E.\n</span>\n\n  \n\n</span>\n\n  In <i>Practical Data Science with Python 3</i>, pages 317–339. Apress,  2019.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://dx.doi.org/10.1007/978-1-4842-4859-1\"\n     onclick=\"log_download('varga-recommendersystems-2019', 'http://dx.doi.org/10.1007/978-1-4842-4859-1', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Recommender Systems [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"http://doi.org/10.1007/978-1-4842-4859-1\"\n     onclick=\"log_download('varga-recommendersystems-2019', 'http://doi.org/10.1007/978-1-4842-4859-1', event.ctrlKey)\"\n     class=\"bibbase doi link\">\n    <span>doi</span></a>\n  &nbsp;\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/varga-recommendersystems-2019\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n  &nbsp;\n  <span class=\"bibbase_stats_paper\" style=\"color: #777;\">\n    <span>5 downloads</span>\n  </span>\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('varga-recommendersystems-2019')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@incollection{varga_recommender_2019,\n\ttitle = {Recommender {Systems}},\n\tisbn = {978-1-4842-4858-4},\n\turl = {http://dx.doi.org/10.1007/978-1-4842-4859-1},\n\tbooktitle = {Practical {Data} {Science} with {Python} 3},\n\tpublisher = {Apress},\n\tauthor = {Varga, Ervin},\n\tyear = {2019},\n\tdoi = {10.1007/978-1-4842-4859-1},\n\tpages = {317--339},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n  <div class=\"bibbase_paper\" id=\"balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019\">\n  <span class=\"bibbase_paper_titleauthoryear\">\n\n  \n  <span class=\"bibbase_paper_title\">\n  \n  \n  \n  <a href=\"http://dspace.ups.edu.ec/handle/123456789/17605\"\n     onclick=\"return trackOutboundLink('publications', 'click', 'balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019', 'http://dspace.ups.edu.ec/handle/123456789/17605', event);\">\n    Estudio del estado del arte de la ciencia de datos aplicada a la neuroeconomía.\n  </a>\n  \n  \n  \n</span>\n\n  <span class=\"bibbase_paper_author\">\n  Balseca Níñez, C. R.;  and Fernández Peñafiel, D. A.\n</span>\n\n  \n\n</span>\n\n  Master's thesis, Universidad Politécnica Salesiana,  2019.\n  <span class=\"note\"></span>\n\n<span class=\"bibbase_note\"></span>\n\n<br class=\"bibbase_paper_content\"/>\n\n<span class=\"bibbase_paper_content dontprint\">\n\n  \n  <a href=\"http://dspace.ups.edu.ec/handle/123456789/17605\"\n     onclick=\"log_download('balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019', 'http://dspace.ups.edu.ec/handle/123456789/17605', event.ctrlKey, event)\">\n    <img src=\"//bibbase.org/img/filetypes/link.svg\"\n\t     alt=\"Estudio del estado del arte de la ciencia de datos aplicada a la neuroeconomía [link]\"\n       title=\"Paper\"\n\t     class=\"bibbase_icon\"\n         ><span class=\"bibbase_icon_text\">Paper</span></a>\n  &nbsp;\n  \n\n  \n\n  \n  <a href=\"//bibbase.org/network/publication/balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019\"\n     class=\"bibbase bibbase_page link\">link</a>\n  &nbsp;\n  \n\n  <a href=\"#\"\n     onclick=\"showBib(event); return false;\"\n     class=\"bibbase bibtex link\">bibtex\n    <i class=\"fa fa-caret-down\"></i></a>\n\n  \n\n  \n\n  <!-- <a class=\"bibbase read link hidden\" -->\n  <!--    href=\"javascript:toggleRead('balsecanez-fernndezpeafiel-estudiodelestadodelartedelacienciadedatosaplicadaalaneuroeconoma-2019')\" -->\n  <!--    title=\"Marking this paper as read adds it to your library on BibBase.\" -->\n  <!--    > -->\n  <!--   <i class=\"fa fa-check\"></i>&nbsp; -->\n  <!--   <span>mark as read</span> -->\n  <!-- </a> -->\n\n  &nbsp;\n  \n  \n\n</span>\n\n<div class=\"well well-small bibbase\" data-type=\"bibtex\"\n     style=\"display:none\">\n  <pre style=\"white-space: pre-wrap;\">@mastersthesis{balseca_ninez_estudio_2019,\n\ttitle = {Estudio del estado del arte de la ciencia de datos aplicada a la neuroeconomía},\n\turl = {http://dspace.ups.edu.ec/handle/123456789/17605},\n\tschool = {Universidad Politécnica Salesiana},\n\tauthor = {Balseca Níñez, Christian Ricardo and Fernández Peñafiel, Daniel Andrés},\n\tyear = {2019},\n}\n\n</pre>\n</div>\n\n\n\n</div>\n\n\n\n\n\n    </div>\n  </div>\n\n\n\n\n  </div>\n\n\n  <!-- Modal -->\n\n  <!-- <iframe id=\"bibbase_network_frame\" -->\n  <!--         src=\"//bibbase.org/network/control\" -->\n  <!--         style=\"display: none;\"> -->\n  <!-- </iframe> -->\n\n</div>\n"}; document.write(bibbase_data.data);