The State of Knowledge Distillation for Classification

The State of Knowledge Distillation for Classification. Ruffy, F. & Chahal, K. December, 2019.

We survey various knowledge distillation (KD) strategies for simple classification tasks and implement a set of techniques that claim state-of-the-art accuracy. Our experiments using standardized model architectures, fixed compute budgets, and consistent training schedules indicate that many of these distillation results are hard to reproduce. This is especially apparent with methods using some form of feature distillation. Further examination reveals a lack of generalizability where these techniques may only succeed for specific architectures and training settings. We observe that appropriately tuned classical distillation in combination with a data augmentation training scheme gives an orthogonal improvement over other techniques. We validate this approach and open-source our code.

@article{ruffy_state_2019,
	title = {The {State} of {Knowledge} {Distillation} for {Classification}},
	url = {https://arxiv.org/abs/1912.10850v1},
	abstract = {We survey various knowledge distillation (KD) strategies for simple
classification tasks and implement a set of techniques that claim
state-of-the-art accuracy. Our experiments using standardized model
architectures, fixed compute budgets, and consistent training schedules
indicate that many of these distillation results are hard to reproduce. This is
especially apparent with methods using some form of feature distillation.
Further examination reveals a lack of generalizability where these techniques
may only succeed for specific architectures and training settings. We observe
that appropriately tuned classical distillation in combination with a data
augmentation training scheme gives an orthogonal improvement over other
techniques. We validate this approach and open-source our code.},
	language = {en},
	urldate = {2020-03-13},
	author = {Ruffy, Fabian and Chahal, Karanbir},
	month = dec,
	year = {2019},
}

Downloads: 0

{"_id":"3jFwgr2RoGZc2NauK","bibbaseid":"ruffy-chahal-thestateofknowledgedistillationforclassification-2019","authorIDs":[],"author_short":["Ruffy, F.","Chahal, K."],"bibdata":{"bibtype":"article","type":"article","title":"The State of Knowledge Distillation for Classification","url":"https://arxiv.org/abs/1912.10850v1","abstract":"We survey various knowledge distillation (KD) strategies for simple classification tasks and implement a set of techniques that claim state-of-the-art accuracy. Our experiments using standardized model architectures, fixed compute budgets, and consistent training schedules indicate that many of these distillation results are hard to reproduce. This is especially apparent with methods using some form of feature distillation. Further examination reveals a lack of generalizability where these techniques may only succeed for specific architectures and training settings. We observe that appropriately tuned classical distillation in combination with a data augmentation training scheme gives an orthogonal improvement over other techniques. We validate this approach and open-source our code.","language":"en","urldate":"2020-03-13","author":[{"propositions":[],"lastnames":["Ruffy"],"firstnames":["Fabian"],"suffixes":[]},{"propositions":[],"lastnames":["Chahal"],"firstnames":["Karanbir"],"suffixes":[]}],"month":"December","year":"2019","bibtex":"@article{ruffy_state_2019,\n\ttitle = {The {State} of {Knowledge} {Distillation} for {Classification}},\n\turl = {https://arxiv.org/abs/1912.10850v1},\n\tabstract = {We survey various knowledge distillation (KD) strategies for simple\nclassification tasks and implement a set of techniques that claim\nstate-of-the-art accuracy. Our experiments using standardized model\narchitectures, fixed compute budgets, and consistent training schedules\nindicate that many of these distillation results are hard to reproduce. This is\nespecially apparent with methods using some form of feature distillation.\nFurther examination reveals a lack of generalizability where these techniques\nmay only succeed for specific architectures and training settings. We observe\nthat appropriately tuned classical distillation in combination with a data\naugmentation training scheme gives an orthogonal improvement over other\ntechniques. We validate this approach and open-source our code.},\n\tlanguage = {en},\n\turldate = {2020-03-13},\n\tauthor = {Ruffy, Fabian and Chahal, Karanbir},\n\tmonth = dec,\n\tyear = {2019},\n}\n\n","author_short":["Ruffy, F.","Chahal, K."],"key":"ruffy_state_2019","id":"ruffy_state_2019","bibbaseid":"ruffy-chahal-thestateofknowledgedistillationforclassification-2019","role":"author","urls":{"Paper":"https://arxiv.org/abs/1912.10850v1"},"downloads":0,"html":""},"bibtype":"article","biburl":"https://bibbase.org/zotero/franciscoperezsorrosal","creationDate":"2021-02-12T21:37:01.710Z","downloads":0,"keywords":[],"search_terms":["state","knowledge","distillation","classification","ruffy","chahal"],"title":"The State of Knowledge Distillation for Classification","year":2019,"dataSources":["qLJ7Ld8T2ZKybATHB"]}