Specifying and Evaluating Quality Metrics for Vision-based Perception Systems. Balakrishnan, A., Puranic, A. G., Qin, X., Dokhanchi, A., Deshmukh, J. V., Ben Amor, H., & Fainekos, G. In 2019 Design, Automation & Test in Europe Conference & Exhibition (DATE), pages 1433–1438, March, 2019. ISSN: 1558-1101doi abstract bibtex Robust perception algorithms are a vital ingredient for autonomous systems such as self-driving vehicles. Checking the correctness of perception algorithms such as those based on deep convolutional neural networks (CNN) is a formidable challenge problem. In this paper, we suggest the use of Timed Quality Temporal Logic (TQTL) as a formal language to express desirable spatio-temporal properties of a perception algorithm processing a video. While perception algorithms are traditionally tested by comparing their performance to ground truth labels, we show how TQTL can be a useful tool to determine quality of perception, and offers an alternative metric that can give useful information, even in the absence of ground truth labels. We demonstrate TQTL monitoring on two popular CNNs: YOLO and SqueezeDet, and give a comparative study of the results obtained for each architecture.
@inproceedings{balakrishnan_specifying_2019,
title = {Specifying and {Evaluating} {Quality} {Metrics} for {Vision}-based {Perception} {Systems}},
doi = {10.23919/DATE.2019.8715114},
abstract = {Robust perception algorithms are a vital ingredient for autonomous systems such as self-driving vehicles. Checking the correctness of perception algorithms such as those based on deep convolutional neural networks (CNN) is a formidable challenge problem. In this paper, we suggest the use of Timed Quality Temporal Logic (TQTL) as a formal language to express desirable spatio-temporal properties of a perception algorithm processing a video. While perception algorithms are traditionally tested by comparing their performance to ground truth labels, we show how TQTL can be a useful tool to determine quality of perception, and offers an alternative metric that can give useful information, even in the absence of ground truth labels. We demonstrate TQTL monitoring on two popular CNNs: YOLO and SqueezeDet, and give a comparative study of the results obtained for each architecture.},
booktitle = {2019 {Design}, {Automation} \& {Test} in {Europe} {Conference} \& {Exhibition} ({DATE})},
author = {Balakrishnan, Anand and Puranic, Aniruddh G. and Qin, Xin and Dokhanchi, Adel and Deshmukh, Jyotirmoy V. and Ben Amor, Heni and Fainekos, Georgios},
month = mar,
year = {2019},
note = {ISSN: 1558-1101},
keywords = {Automobiles, Autonomous vehicles, Convolutional neural networks, Cost accounting, Image processing, Measurement, Monitoring, Perception, Quality Metrics, Real-time systems, Semantics, Temporal Logic},
pages = {1433--1438},
}
Downloads: 0
{"_id":"XxzhDkBburJAm5QAb","bibbaseid":"balakrishnan-puranic-qin-dokhanchi-deshmukh-benamor-fainekos-specifyingandevaluatingqualitymetricsforvisionbasedperceptionsystems-2019","author_short":["Balakrishnan, A.","Puranic, A. G.","Qin, X.","Dokhanchi, A.","Deshmukh, J. V.","Ben Amor, H.","Fainekos, G."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","title":"Specifying and Evaluating Quality Metrics for Vision-based Perception Systems","doi":"10.23919/DATE.2019.8715114","abstract":"Robust perception algorithms are a vital ingredient for autonomous systems such as self-driving vehicles. Checking the correctness of perception algorithms such as those based on deep convolutional neural networks (CNN) is a formidable challenge problem. In this paper, we suggest the use of Timed Quality Temporal Logic (TQTL) as a formal language to express desirable spatio-temporal properties of a perception algorithm processing a video. While perception algorithms are traditionally tested by comparing their performance to ground truth labels, we show how TQTL can be a useful tool to determine quality of perception, and offers an alternative metric that can give useful information, even in the absence of ground truth labels. We demonstrate TQTL monitoring on two popular CNNs: YOLO and SqueezeDet, and give a comparative study of the results obtained for each architecture.","booktitle":"2019 Design, Automation & Test in Europe Conference & Exhibition (DATE)","author":[{"propositions":[],"lastnames":["Balakrishnan"],"firstnames":["Anand"],"suffixes":[]},{"propositions":[],"lastnames":["Puranic"],"firstnames":["Aniruddh","G."],"suffixes":[]},{"propositions":[],"lastnames":["Qin"],"firstnames":["Xin"],"suffixes":[]},{"propositions":[],"lastnames":["Dokhanchi"],"firstnames":["Adel"],"suffixes":[]},{"propositions":[],"lastnames":["Deshmukh"],"firstnames":["Jyotirmoy","V."],"suffixes":[]},{"propositions":[],"lastnames":["Ben","Amor"],"firstnames":["Heni"],"suffixes":[]},{"propositions":[],"lastnames":["Fainekos"],"firstnames":["Georgios"],"suffixes":[]}],"month":"March","year":"2019","note":"ISSN: 1558-1101","keywords":"Automobiles, Autonomous vehicles, Convolutional neural networks, Cost accounting, Image processing, Measurement, Monitoring, Perception, Quality Metrics, Real-time systems, Semantics, Temporal Logic","pages":"1433–1438","bibtex":"@inproceedings{balakrishnan_specifying_2019,\n\ttitle = {Specifying and {Evaluating} {Quality} {Metrics} for {Vision}-based {Perception} {Systems}},\n\tdoi = {10.23919/DATE.2019.8715114},\n\tabstract = {Robust perception algorithms are a vital ingredient for autonomous systems such as self-driving vehicles. Checking the correctness of perception algorithms such as those based on deep convolutional neural networks (CNN) is a formidable challenge problem. In this paper, we suggest the use of Timed Quality Temporal Logic (TQTL) as a formal language to express desirable spatio-temporal properties of a perception algorithm processing a video. While perception algorithms are traditionally tested by comparing their performance to ground truth labels, we show how TQTL can be a useful tool to determine quality of perception, and offers an alternative metric that can give useful information, even in the absence of ground truth labels. We demonstrate TQTL monitoring on two popular CNNs: YOLO and SqueezeDet, and give a comparative study of the results obtained for each architecture.},\n\tbooktitle = {2019 {Design}, {Automation} \\& {Test} in {Europe} {Conference} \\& {Exhibition} ({DATE})},\n\tauthor = {Balakrishnan, Anand and Puranic, Aniruddh G. and Qin, Xin and Dokhanchi, Adel and Deshmukh, Jyotirmoy V. and Ben Amor, Heni and Fainekos, Georgios},\n\tmonth = mar,\n\tyear = {2019},\n\tnote = {ISSN: 1558-1101},\n\tkeywords = {Automobiles, Autonomous vehicles, Convolutional neural networks, Cost accounting, Image processing, Measurement, Monitoring, Perception, Quality Metrics, Real-time systems, Semantics, Temporal Logic},\n\tpages = {1433--1438},\n}\n\n","author_short":["Balakrishnan, A.","Puranic, A. G.","Qin, X.","Dokhanchi, A.","Deshmukh, J. V.","Ben Amor, H.","Fainekos, G."],"key":"balakrishnan_specifying_2019","id":"balakrishnan_specifying_2019","bibbaseid":"balakrishnan-puranic-qin-dokhanchi-deshmukh-benamor-fainekos-specifyingandevaluatingqualitymetricsforvisionbasedperceptionsystems-2019","role":"author","urls":{},"keyword":["Automobiles","Autonomous vehicles","Convolutional neural networks","Cost accounting","Image processing","Measurement","Monitoring","Perception","Quality Metrics","Real-time systems","Semantics","Temporal Logic"],"metadata":{"authorlinks":{}}},"bibtype":"inproceedings","biburl":"https://api.zotero.org/users/6738928/collections/GUL2T6VV/items?key=0xkjXd4tKtTWLWQfNOnUSt39&format=bibtex&limit=100","dataSources":["LEBX9R2q2PGo53py4"],"keywords":["automobiles","autonomous vehicles","convolutional neural networks","cost accounting","image processing","measurement","monitoring","perception","quality metrics","real-time systems","semantics","temporal logic"],"search_terms":["specifying","evaluating","quality","metrics","vision","based","perception","systems","balakrishnan","puranic","qin","dokhanchi","deshmukh","ben amor","fainekos"],"title":"Specifying and Evaluating Quality Metrics for Vision-based Perception Systems","year":2019}