StreamDFP: A General Stream Mining Framework for Adaptive Disk Failure Prediction. Han, S., Lee, P. P. C., Shen, Z., He, C., Liu, Y., & Huang, T. IEEE Transactions on Computers, 2022. Conference Name: IEEE Transactions on Computersdoi abstract bibtex We explore machine learning for accurately predicting imminent disk failures and hence providing proactive fault tolerance for modern large-scale storage systems. Current disk failure prediction approaches are mostly offline and assume that the disk logs required for training learning models are available a priori. However, disk logs are often continuously generated as an evolving data stream, in which the statistical patterns vary over time (also known as concept drift). Such a challenge motivates the need of online techniques that perform training and prediction on the incoming stream of disk logs in real time, while being adaptive to concept drift. We first measure and demonstrate the existence of concept drift on various disk models from Backblaze and Alibaba Cloud. Motivated by our study, we design STREAMDFP, a general stream mining framework for disk failure prediction with concept-drift adaption based on three key techniques, namely online labeling, concept-drift-aware training, and general prediction, with a primary objective of supporting various machine learning algorithms. We extend STREAMDFP to support online transfer learning for minority disk models with concept-drift adaptation. Our evaluation shows that STREAMDFP improves the prediction accuracy significantly compared to without concept-drift adaptation under various settings, and achieves reasonably high stream processing performance.
@article{han_streamdfp_2022,
title = {{StreamDFP}: {A} {General} {Stream} {Mining} {Framework} for {Adaptive} {Disk} {Failure} {Prediction}},
issn = {1557-9956},
shorttitle = {{StreamDFP}},
doi = {10.1109/TC.2022.3160365},
abstract = {We explore machine learning for accurately predicting imminent disk failures and hence providing proactive fault tolerance for modern large-scale storage systems. Current disk failure prediction approaches are mostly offline and assume that the disk logs required for training learning models are available a priori. However, disk logs are often continuously generated as an evolving data stream, in which the statistical patterns vary over time (also known as concept drift). Such a challenge motivates the need of online techniques that perform training and prediction on the incoming stream of disk logs in real time, while being adaptive to concept drift. We first measure and demonstrate the existence of concept drift on various disk models from Backblaze and Alibaba Cloud. Motivated by our study, we design STREAMDFP, a general stream mining framework for disk failure prediction with concept-drift adaption based on three key techniques, namely online labeling, concept-drift-aware training, and general prediction, with a primary objective of supporting various machine learning algorithms. We extend STREAMDFP to support online transfer learning for minority disk models with concept-drift adaptation. Our evaluation shows that STREAMDFP improves the prediction accuracy significantly compared to without concept-drift adaptation under various settings, and achieves reasonably high stream processing performance.},
journal = {IEEE Transactions on Computers},
author = {Han, Shujie and Lee, Patrick P. C. and Shen, Zhirong and He, Cheng and Liu, Yi and Huang, Tao},
year = {2022},
note = {Conference Name: IEEE Transactions on Computers},
keywords = {Adaptation models, Machine learning algorithms, Prediction algorithms, Predictive models, Production, Random forests, Training, and online transfer learning, concept drift, disk failure prediction, stream mining},
pages = {1--1},
}
Downloads: 0
{"_id":"xZjkhECWGYryXDXfZ","bibbaseid":"han-lee-shen-he-liu-huang-streamdfpageneralstreamminingframeworkforadaptivediskfailureprediction-2022","author_short":["Han, S.","Lee, P. P. C.","Shen, Z.","He, C.","Liu, Y.","Huang, T."],"bibdata":{"bibtype":"article","type":"article","title":"StreamDFP: A General Stream Mining Framework for Adaptive Disk Failure Prediction","issn":"1557-9956","shorttitle":"StreamDFP","doi":"10.1109/TC.2022.3160365","abstract":"We explore machine learning for accurately predicting imminent disk failures and hence providing proactive fault tolerance for modern large-scale storage systems. Current disk failure prediction approaches are mostly offline and assume that the disk logs required for training learning models are available a priori. However, disk logs are often continuously generated as an evolving data stream, in which the statistical patterns vary over time (also known as concept drift). Such a challenge motivates the need of online techniques that perform training and prediction on the incoming stream of disk logs in real time, while being adaptive to concept drift. We first measure and demonstrate the existence of concept drift on various disk models from Backblaze and Alibaba Cloud. Motivated by our study, we design STREAMDFP, a general stream mining framework for disk failure prediction with concept-drift adaption based on three key techniques, namely online labeling, concept-drift-aware training, and general prediction, with a primary objective of supporting various machine learning algorithms. We extend STREAMDFP to support online transfer learning for minority disk models with concept-drift adaptation. Our evaluation shows that STREAMDFP improves the prediction accuracy significantly compared to without concept-drift adaptation under various settings, and achieves reasonably high stream processing performance.","journal":"IEEE Transactions on Computers","author":[{"propositions":[],"lastnames":["Han"],"firstnames":["Shujie"],"suffixes":[]},{"propositions":[],"lastnames":["Lee"],"firstnames":["Patrick","P.","C."],"suffixes":[]},{"propositions":[],"lastnames":["Shen"],"firstnames":["Zhirong"],"suffixes":[]},{"propositions":[],"lastnames":["He"],"firstnames":["Cheng"],"suffixes":[]},{"propositions":[],"lastnames":["Liu"],"firstnames":["Yi"],"suffixes":[]},{"propositions":[],"lastnames":["Huang"],"firstnames":["Tao"],"suffixes":[]}],"year":"2022","note":"Conference Name: IEEE Transactions on Computers","keywords":"Adaptation models, Machine learning algorithms, Prediction algorithms, Predictive models, Production, Random forests, Training, and online transfer learning, concept drift, disk failure prediction, stream mining","pages":"1–1","bibtex":"@article{han_streamdfp_2022,\n\ttitle = {{StreamDFP}: {A} {General} {Stream} {Mining} {Framework} for {Adaptive} {Disk} {Failure} {Prediction}},\n\tissn = {1557-9956},\n\tshorttitle = {{StreamDFP}},\n\tdoi = {10.1109/TC.2022.3160365},\n\tabstract = {We explore machine learning for accurately predicting imminent disk failures and hence providing proactive fault tolerance for modern large-scale storage systems. Current disk failure prediction approaches are mostly offline and assume that the disk logs required for training learning models are available a priori. However, disk logs are often continuously generated as an evolving data stream, in which the statistical patterns vary over time (also known as concept drift). Such a challenge motivates the need of online techniques that perform training and prediction on the incoming stream of disk logs in real time, while being adaptive to concept drift. We first measure and demonstrate the existence of concept drift on various disk models from Backblaze and Alibaba Cloud. Motivated by our study, we design STREAMDFP, a general stream mining framework for disk failure prediction with concept-drift adaption based on three key techniques, namely online labeling, concept-drift-aware training, and general prediction, with a primary objective of supporting various machine learning algorithms. We extend STREAMDFP to support online transfer learning for minority disk models with concept-drift adaptation. Our evaluation shows that STREAMDFP improves the prediction accuracy significantly compared to without concept-drift adaptation under various settings, and achieves reasonably high stream processing performance.},\n\tjournal = {IEEE Transactions on Computers},\n\tauthor = {Han, Shujie and Lee, Patrick P. C. and Shen, Zhirong and He, Cheng and Liu, Yi and Huang, Tao},\n\tyear = {2022},\n\tnote = {Conference Name: IEEE Transactions on Computers},\n\tkeywords = {Adaptation models, Machine learning algorithms, Prediction algorithms, Predictive models, Production, Random forests, Training, and online transfer learning, concept drift, disk failure prediction, stream mining},\n\tpages = {1--1},\n}\n\n","author_short":["Han, S.","Lee, P. P. C.","Shen, Z.","He, C.","Liu, Y.","Huang, T."],"key":"han_streamdfp_2022","id":"han_streamdfp_2022","bibbaseid":"han-lee-shen-he-liu-huang-streamdfpageneralstreamminingframeworkforadaptivediskfailureprediction-2022","role":"author","urls":{},"keyword":["Adaptation models","Machine learning algorithms","Prediction algorithms","Predictive models","Production","Random forests","Training","and online transfer learning","concept drift","disk failure prediction","stream mining"],"metadata":{"authorlinks":{}}},"bibtype":"article","biburl":"https://bibbase.org/zotero/mh_lenguyen","dataSources":["iwKepCrWBps7ojhDx"],"keywords":["adaptation models","machine learning algorithms","prediction algorithms","predictive models","production","random forests","training","and online transfer learning","concept drift","disk failure prediction","stream mining"],"search_terms":["streamdfp","general","stream","mining","framework","adaptive","disk","failure","prediction","han","lee","shen","he","liu","huang"],"title":"StreamDFP: A General Stream Mining Framework for Adaptive Disk Failure Prediction","year":2022}