Selecting the Number of Clusters $K$ with a Stability Trade-off: an Internal Validation Criterion. Mourer, A., Forest, F., Lebbah, M., Azzag, H., & Lacaille, J. In PAKDD, Osaka, Japan, May, 2023. arXiv:2006.08530 [cs, stat]
![Selecting the Number of Clusters $K$ with a Stability Trade-off: an Internal Validation Criterion [link]](https://bibbase.org/img/filetypes/link.svg "link")
Link ![Selecting the Number of Clusters $K$ with a Stability Trade-off: an Internal Validation Criterion [pdf]](https://bibbase.org/img/filetypes/pdf.svg "pdf")
Paper Code doi abstract bibtex 1 download
Link Paper Code doi abstract bibtex 1 download Model selection is a major challenge in non-parametric clustering. There is no universally admitted way to evaluate clustering results for the obvious reason that no ground truth is available. The difficulty to find a universal evaluation criterion is a consequence of the ill-defined objective of clustering. In this perspective, clustering stability has emerged as a natural and model-agnostic principle: an algorithm should find stable structures in the data. If data sets are repeatedly sampled from the same underlying distribution, an algorithm should find similar partitions. However, stability alone is not well-suited to determine the number of clusters. For instance, it is unable to detect if the number of clusters is too small. We propose a new principle: a good clustering should be stable, and within each cluster, there should exist no stable partition. This principle leads to a novel clustering validation criterion based on between-cluster and within-cluster stability, overcoming limitations of previous stability-based methods. We empirically demonstrate the effectiveness of our criterion to select the number of clusters and compare it with existing methods. Code is available at https://github.com/FlorentF9/skstab.
@inproceedings{mourer2023selecting,
address = {Osaka, Japan},
title = {Selecting the {Number} of {Clusters} $K$ with a {Stability} {Trade}-off: an {Internal} {Validation} {Criterion}},
copyright = {All rights reserved},
shorttitle = {Selecting the {Number} of {Clusters} $K$ with a {Stability} {Trade}-off},
doi = {10.48550/arXiv.2006.08530},
abstract = {Model selection is a major challenge in non-parametric clustering. There is no universally admitted way to evaluate clustering results for the obvious reason that no ground truth is available. The difficulty to find a universal evaluation criterion is a consequence of the ill-defined objective of clustering. In this perspective, clustering stability has emerged as a natural and model-agnostic principle: an algorithm should find stable structures in the data. If data sets are repeatedly sampled from the same underlying distribution, an algorithm should find similar partitions. However, stability alone is not well-suited to determine the number of clusters. For instance, it is unable to detect if the number of clusters is too small. We propose a new principle: a good clustering should be stable, and within each cluster, there should exist no stable partition. This principle leads to a novel clustering validation criterion based on between-cluster and within-cluster stability, overcoming limitations of previous stability-based methods. We empirically demonstrate the effectiveness of our criterion to select the number of clusters and compare it with existing methods. Code is available at https://github.com/FlorentF9/skstab.},
booktitle = {{PAKDD}},
author = {Mourer, Alex and Forest, Florent and Lebbah, Mustapha and Azzag, Hanane and Lacaille, Jérôme},
month = may,
year = {2023},
note = {arXiv:2006.08530 [cs, stat]},
keywords = {validity index,clustering,model selection,stability analysis},
url_Link = {http://arxiv.org/abs/2006.08530},
url_Paper = {http://arxiv.org/pdf/2006.08530.pdf},
url_Code = {https://github.com/FlorentF9/skstab},
bibbase_note = {<img src="assets/img/papers/stadion.png">}
}Downloads: 1
{"_id":"5WDZK9aGKjumra8pj","bibbaseid":"mourer-forest-lebbah-azzag-lacaille-selectingthenumberofclusterskwithastabilitytradeoffaninternalvalidationcriterion-2023","author_short":["Mourer, A.","Forest, F.","Lebbah, M.","Azzag, H.","Lacaille, J."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"Osaka, Japan","title":"Selecting the Number of Clusters $K$ with a Stability Trade-off: an Internal Validation Criterion","copyright":"All rights reserved","shorttitle":"Selecting the Number of Clusters $K$ with a Stability Trade-off","doi":"10.48550/arXiv.2006.08530","abstract":"Model selection is a major challenge in non-parametric clustering. There is no universally admitted way to evaluate clustering results for the obvious reason that no ground truth is available. The difficulty to find a universal evaluation criterion is a consequence of the ill-defined objective of clustering. In this perspective, clustering stability has emerged as a natural and model-agnostic principle: an algorithm should find stable structures in the data. If data sets are repeatedly sampled from the same underlying distribution, an algorithm should find similar partitions. However, stability alone is not well-suited to determine the number of clusters. For instance, it is unable to detect if the number of clusters is too small. We propose a new principle: a good clustering should be stable, and within each cluster, there should exist no stable partition. This principle leads to a novel clustering validation criterion based on between-cluster and within-cluster stability, overcoming limitations of previous stability-based methods. We empirically demonstrate the effectiveness of our criterion to select the number of clusters and compare it with existing methods. Code is available at https://github.com/FlorentF9/skstab.","booktitle":"PAKDD","author":[{"propositions":[],"lastnames":["Mourer"],"firstnames":["Alex"],"suffixes":[]},{"propositions":[],"lastnames":["Forest"],"firstnames":["Florent"],"suffixes":[]},{"propositions":[],"lastnames":["Lebbah"],"firstnames":["Mustapha"],"suffixes":[]},{"propositions":[],"lastnames":["Azzag"],"firstnames":["Hanane"],"suffixes":[]},{"propositions":[],"lastnames":["Lacaille"],"firstnames":["Jérôme"],"suffixes":[]}],"month":"May","year":"2023","note":"arXiv:2006.08530 [cs, stat]","keywords":"validity index,clustering,model selection,stability analysis","url_link":"http://arxiv.org/abs/2006.08530","url_paper":"http://arxiv.org/pdf/2006.08530.pdf","url_code":"https://github.com/FlorentF9/skstab","bibbase_note":"<img src=\"assets/img/papers/stadion.png\">","bibtex":"@inproceedings{mourer2023selecting,\n address = {Osaka, Japan},\n title = {Selecting the {Number} of {Clusters} $K$ with a {Stability} {Trade}-off: an {Internal} {Validation} {Criterion}},\n copyright = {All rights reserved},\n shorttitle = {Selecting the {Number} of {Clusters} $K$ with a {Stability} {Trade}-off},\n doi = {10.48550/arXiv.2006.08530},\n abstract = {Model selection is a major challenge in non-parametric clustering. There is no universally admitted way to evaluate clustering results for the obvious reason that no ground truth is available. The difficulty to find a universal evaluation criterion is a consequence of the ill-defined objective of clustering. In this perspective, clustering stability has emerged as a natural and model-agnostic principle: an algorithm should find stable structures in the data. If data sets are repeatedly sampled from the same underlying distribution, an algorithm should find similar partitions. However, stability alone is not well-suited to determine the number of clusters. For instance, it is unable to detect if the number of clusters is too small. We propose a new principle: a good clustering should be stable, and within each cluster, there should exist no stable partition. This principle leads to a novel clustering validation criterion based on between-cluster and within-cluster stability, overcoming limitations of previous stability-based methods. We empirically demonstrate the effectiveness of our criterion to select the number of clusters and compare it with existing methods. Code is available at https://github.com/FlorentF9/skstab.},\n booktitle = {{PAKDD}},\n author = {Mourer, Alex and Forest, Florent and Lebbah, Mustapha and Azzag, Hanane and Lacaille, Jérôme},\n month = may,\n year = {2023},\n note = {arXiv:2006.08530 [cs, stat]},\n keywords = {validity index,clustering,model selection,stability analysis},\n url_Link = {http://arxiv.org/abs/2006.08530},\n url_Paper = {http://arxiv.org/pdf/2006.08530.pdf},\n\turl_Code = {https://github.com/FlorentF9/skstab},\n bibbase_note = {<img src=\"assets/img/papers/stadion.png\">}\n}\n\n","author_short":["Mourer, A.","Forest, F.","Lebbah, M.","Azzag, H.","Lacaille, J."],"key":"mourer2023selecting","id":"mourer2023selecting","bibbaseid":"mourer-forest-lebbah-azzag-lacaille-selectingthenumberofclusterskwithastabilitytradeoffaninternalvalidationcriterion-2023","role":"author","urls":{" link":"http://arxiv.org/abs/2006.08530"," paper":"http://arxiv.org/pdf/2006.08530.pdf"," code":"https://github.com/FlorentF9/skstab"},"keyword":["validity index","clustering","model selection","stability analysis"],"metadata":{"authorlinks":{}},"downloads":1},"bibtype":"inproceedings","biburl":"https://florentfo.rest/files/publications.bib","dataSources":["2puawT8ZAQyYRypA3","6rNfa4Kp6dL5sGmf5","xH8ySTsEPTLou9gyR","pBkCjKbyeirr5jeAd","DgnR6pzJ98ZEp97PW"],"keywords":["validity index","clustering","model selection","stability analysis"],"search_terms":["selecting","number","clusters","stability","trade","internal","validation","criterion","mourer","forest","lebbah","azzag","lacaille"],"title":"Selecting the Number of Clusters $K$ with a Stability Trade-off: an Internal Validation Criterion","year":2023,"downloads":1}