On rater reliability and agreement based dynamic active learning

On rater reliability and agreement based dynamic active learning. Zhang, Y., Coutinho, E., Schuller, B., Zhang, Z., & Adam, M. In 2015 International Conference on Affective Computing and Intelligent Interaction, ACII 2015, pages 70-76, 9, 2015. IEEE.

Website doi abstract bibtex

In this paper, we propose two novel Dynamic Active Learning (DAL) methods with the aim of ultimately reducing the costly human labelling work for subjective tasks such as speech emotion recognition. Compared to conventional Active Learning (AL) algorithms, the proposed DAL approaches employ a highly efficient adaptive query strategy that minimises the number of annotations through three advancements. First, we shift from the standard majority voting procedure, in which unlabelled instances are annotated by a fixed number of raters, to an agreement-based annotation technique that dynamically determines how many human annotators are required to label a selected instance. Second, we introduce the concept of the order-based DAL algorithm by considering rater reliability and inter-rater agreement. Third, a highly dynamic development trend is successfully implemented by upgrading the agreement levels depending on the prediction uncertainty. In extensive experiments on standardised test-beds, we show that the new dynamic methods significantly improve the efficiency of the existing AL algorithms by reducing human labelling effort up to 85.41%, while achieving the same classification accuracy. Thus, the enhanced DAL derivations opens up high-potential research directions for the utmost exploitation of unlabelled data.

@inproceedings{
 title = {On rater reliability and agreement based dynamic active learning},
 type = {inproceedings},
 year = {2015},
 keywords = {article,conference},
 pages = {70-76},
 websites = {http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcApp=PARTNER_APP&SrcAuth=LinksAMR&KeyUT=WOS:000377887000011&DestLinkType=FullRecord&DestApp=ALL_WOS&UsrCustomerID=f3ec48df247ee1138ccd8d3ba59bacc2,http://ieeexplore.ieee.org/document/7344},
 month = {9},
 publisher = {IEEE},
 id = {38dc393a-f322-3f71-9db6-0bf17a81191b},
 created = {2020-05-29T11:51:37.237Z},
 file_attached = {true},
 profile_id = {ffa9027c-806a-3827-93a1-02c42eb146a1},
 last_modified = {2020-06-08T17:31:31.875Z},
 read = {false},
 starred = {false},
 authored = {true},
 confirmed = {true},
 hidden = {false},
 citation_key = {zhang2015onlearning},
 source_type = {inproceedings},
 folder_uuids = {aac08d0d-38e7-4f4e-a381-5271c5c099ce},
 private_publication = {false},
 abstract = {In this paper, we propose two novel Dynamic Active Learning (DAL) methods with the aim of ultimately reducing the costly human labelling work for subjective tasks such as speech emotion recognition. Compared to conventional Active Learning (AL) algorithms, the proposed DAL approaches employ a highly efficient adaptive query strategy that minimises the number of annotations through three advancements. First, we shift from the standard majority voting procedure, in which unlabelled instances are annotated by a fixed number of raters, to an agreement-based annotation technique that dynamically determines how many human annotators are required to label a selected instance. Second, we introduce the concept of the order-based DAL algorithm by considering rater reliability and inter-rater agreement. Third, a highly dynamic development trend is successfully implemented by upgrading the agreement levels depending on the prediction uncertainty. In extensive experiments on standardised test-beds, we show that the new dynamic methods significantly improve the efficiency of the existing AL algorithms by reducing human labelling effort up to 85.41%, while achieving the same classification accuracy. Thus, the enhanced DAL derivations opens up high-potential research directions for the utmost exploitation of unlabelled data.},
 bibtype = {inproceedings},
 author = {Zhang, Yue and Coutinho, Eduardo and Schuller, Bjorn and Zhang, Zixing and Adam, Michael},
 doi = {10.1109/ACII.2015.7344553},
 booktitle = {2015 International Conference on Affective Computing and Intelligent Interaction, ACII 2015}
}

Downloads: 0

{"_id":"yM5KuCihQrMgHLuRg","bibbaseid":"zhang-coutinho-schuller-zhang-adam-onraterreliabilityandagreementbaseddynamicactivelearning-2015","downloads":0,"creationDate":"2017-04-06T12:03:59.104Z","title":"On rater reliability and agreement based dynamic active learning","author_short":["Zhang, Y.","Coutinho, E.","Schuller, B.","Zhang, Z.","Adam, M."],"year":2015,"bibtype":"inproceedings","biburl":"https://bibbase.org/service/mendeley/ffa9027c-806a-3827-93a1-02c42eb146a1","bibdata":{"title":"On rater reliability and agreement based dynamic active learning","type":"inproceedings","year":"2015","keywords":"article,conference","pages":"70-76","websites":"http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcApp=PARTNER_APP&SrcAuth=LinksAMR&KeyUT=WOS:000377887000011&DestLinkType=FullRecord&DestApp=ALL_WOS&UsrCustomerID=f3ec48df247ee1138ccd8d3ba59bacc2,http://ieeexplore.ieee.org/document/7344","month":"9","publisher":"IEEE","id":"38dc393a-f322-3f71-9db6-0bf17a81191b","created":"2020-05-29T11:51:37.237Z","file_attached":"true","profile_id":"ffa9027c-806a-3827-93a1-02c42eb146a1","last_modified":"2020-06-08T17:31:31.875Z","read":false,"starred":false,"authored":"true","confirmed":"true","hidden":false,"citation_key":"zhang2015onlearning","source_type":"inproceedings","folder_uuids":"aac08d0d-38e7-4f4e-a381-5271c5c099ce","private_publication":false,"abstract":"In this paper, we propose two novel Dynamic Active Learning (DAL) methods with the aim of ultimately reducing the costly human labelling work for subjective tasks such as speech emotion recognition. Compared to conventional Active Learning (AL) algorithms, the proposed DAL approaches employ a highly efficient adaptive query strategy that minimises the number of annotations through three advancements. First, we shift from the standard majority voting procedure, in which unlabelled instances are annotated by a fixed number of raters, to an agreement-based annotation technique that dynamically determines how many human annotators are required to label a selected instance. Second, we introduce the concept of the order-based DAL algorithm by considering rater reliability and inter-rater agreement. Third, a highly dynamic development trend is successfully implemented by upgrading the agreement levels depending on the prediction uncertainty. In extensive experiments on standardised test-beds, we show that the new dynamic methods significantly improve the efficiency of the existing AL algorithms by reducing human labelling effort up to 85.41%, while achieving the same classification accuracy. Thus, the enhanced DAL derivations opens up high-potential research directions for the utmost exploitation of unlabelled data.","bibtype":"inproceedings","author":"Zhang, Yue and Coutinho, Eduardo and Schuller, Bjorn and Zhang, Zixing and Adam, Michael","doi":"10.1109/ACII.2015.7344553","booktitle":"2015 International Conference on Affective Computing and Intelligent Interaction, ACII 2015","bibtex":"@inproceedings{\n title = {On rater reliability and agreement based dynamic active learning},\n type = {inproceedings},\n year = {2015},\n keywords = {article,conference},\n pages = {70-76},\n websites = {http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcApp=PARTNER_APP&SrcAuth=LinksAMR&KeyUT=WOS:000377887000011&DestLinkType=FullRecord&DestApp=ALL_WOS&UsrCustomerID=f3ec48df247ee1138ccd8d3ba59bacc2,http://ieeexplore.ieee.org/document/7344},\n month = {9},\n publisher = {IEEE},\n id = {38dc393a-f322-3f71-9db6-0bf17a81191b},\n created = {2020-05-29T11:51:37.237Z},\n file_attached = {true},\n profile_id = {ffa9027c-806a-3827-93a1-02c42eb146a1},\n last_modified = {2020-06-08T17:31:31.875Z},\n read = {false},\n starred = {false},\n authored = {true},\n confirmed = {true},\n hidden = {false},\n citation_key = {zhang2015onlearning},\n source_type = {inproceedings},\n folder_uuids = {aac08d0d-38e7-4f4e-a381-5271c5c099ce},\n private_publication = {false},\n abstract = {In this paper, we propose two novel Dynamic Active Learning (DAL) methods with the aim of ultimately reducing the costly human labelling work for subjective tasks such as speech emotion recognition. Compared to conventional Active Learning (AL) algorithms, the proposed DAL approaches employ a highly efficient adaptive query strategy that minimises the number of annotations through three advancements. First, we shift from the standard majority voting procedure, in which unlabelled instances are annotated by a fixed number of raters, to an agreement-based annotation technique that dynamically determines how many human annotators are required to label a selected instance. Second, we introduce the concept of the order-based DAL algorithm by considering rater reliability and inter-rater agreement. Third, a highly dynamic development trend is successfully implemented by upgrading the agreement levels depending on the prediction uncertainty. In extensive experiments on standardised test-beds, we show that the new dynamic methods significantly improve the efficiency of the existing AL algorithms by reducing human labelling effort up to 85.41%, while achieving the same classification accuracy. Thus, the enhanced DAL derivations opens up high-potential research directions for the utmost exploitation of unlabelled data.},\n bibtype = {inproceedings},\n author = {Zhang, Yue and Coutinho, Eduardo and Schuller, Bjorn and Zhang, Zixing and Adam, Michael},\n doi = {10.1109/ACII.2015.7344553},\n booktitle = {2015 International Conference on Affective Computing and Intelligent Interaction, ACII 2015}\n}","author_short":["Zhang, Y.","Coutinho, E.","Schuller, B.","Zhang, Z.","Adam, M."],"urls":{"Website":"http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcApp=PARTNER_APP&SrcAuth=LinksAMR&KeyUT=WOS:000377887000011&DestLinkType=FullRecord&DestApp=ALL_WOS&UsrCustomerID=f3ec48df247ee1138ccd8d3ba59bacc2,http://ieeexplore.ieee.org/document/7344"},"biburl":"https://bibbase.org/service/mendeley/ffa9027c-806a-3827-93a1-02c42eb146a1","bibbaseid":"zhang-coutinho-schuller-zhang-adam-onraterreliabilityandagreementbaseddynamicactivelearning-2015","role":"author","keyword":["article","conference"],"metadata":{"authorlinks":{"coutinho, e":"https://amlab.liverpool.ac.uk/publications.html"}},"downloads":0},"search_terms":["rater","reliability","agreement","based","dynamic","active","learning","zhang","coutinho","schuller","zhang","adam"],"keywords":["article","conference"],"authorIDs":["58e62eafe417243f58000008","6Z276FjCJdxu2qttA","HmyGQqdmpxNriPHXS","LxZfJ4SZafxuJzeKG","S2CzJg64apkJsPsHA","hzjyyz7CdSRZyRptY","mo4CFXJ7ukAMT9nho","nxyiocLrG567rGYAm","oenDqaFMmkBMZtjuK","tE6ysWmRvSH5uoPFW"],"dataSources":["Tcd3cXtdQsiKHPZsW","ya2CyA73rpZseyrZ8","YqW8pMoihb7JazZcx"]}