@article{krotovUnsupervisedLearningCompeting2019,
  title = {Unsupervised Learning by Competing Hidden Units},
  author = {Krotov, Dmitry and Hopfield, John J.},
  date = {2019-04-16},
  journaltitle = {Proceedings of the National Academy of Sciences},
  shortjournal = {PNAS},
  volume = {116},
  pages = {7723--7731},
  issn = {0027-8424, 1091-6490},
  doi = {10.1073/pnas.1820458116},
  url = {https://doi.org/10.1073/pnas.1820458116},
  urldate = {2019-04-18},
  abstract = {[Significance]
Despite great success of deep learning a question remains to what extent the computational properties of deep neural networks are similar to those of the human brain. The particularly nonbiological aspect of deep learning is the supervised training process with the backpropagation algorithm, which requires massive amounts of labeled data, and a nonlocal learning rule for changing the synapse strengths. This paper describes a learning algorithm that does not suffer from these two problems. It learns the weights of the lower layer of neural networks in a completely unsupervised fashion. The entire algorithm utilizes local learning rules which have conceptual biological plausibility.

[Abstract]
It is widely believed that end-to-end training with the backpropagation algorithm is essential for learning good feature detectors in early layers of artificial neural networks, so that these detectors are useful for the task performed by the higher layers of that neural network. At the same time, the traditional form of backpropagation is biologically implausible. In the present paper we propose an unusual learning rule, which has a degree of biological plausibility and which is motivated by Hebb’s idea that change of the synapse strength should be local—i.e., should depend only on the activities of the pre- and postsynaptic neurons. We design a learning algorithm that utilizes global inhibition in the hidden layer and is capable of learning early feature detectors in a completely unsupervised way. These learned lower-layer feature detectors can be used to train higher-layer weights in a usual supervised way so that the performance of the full network is comparable to the performance of standard feedforward networks trained end-to-end with a backpropagation algorithm on simple tasks.},
  eprint = {30926658},
  eprinttype = {pmid},
  keywords = {~INRMM-MiD:z-3D6W95X4,artificial-intelligence,artificial-neural-networks,back-propagation-networks,biological-deep-learning,computational-science,computational-science-automation,deep-learning,machine-learning,unsupervised-training},
  langid = {english},
  number = {16}
}

{"_id":"XGAvFskLeAZHggWPH","bibbaseid":"krotov-hopfield-unsupervisedlearningbycompetinghiddenunits","authorIDs":[],"author_short":["Krotov, D.","Hopfield, J. J."],"bibdata":{"bibtype":"article","type":"article","title":"Unsupervised Learning by Competing Hidden Units","author":[{"propositions":[],"lastnames":["Krotov"],"firstnames":["Dmitry"],"suffixes":[]},{"propositions":[],"lastnames":["Hopfield"],"firstnames":["John","J."],"suffixes":[]}],"date":"2019-04-16","journaltitle":"Proceedings of the National Academy of Sciences","shortjournal":"PNAS","volume":"116","pages":"7723–7731","issn":"0027-8424, 1091-6490","doi":"10.1073/pnas.1820458116","url":"https://doi.org/10.1073/pnas.1820458116","urldate":"2019-04-18","abstract":"[Significance] Despite great success of deep learning a question remains to what extent the computational properties of deep neural networks are similar to those of the human brain. The particularly nonbiological aspect of deep learning is the supervised training process with the backpropagation algorithm, which requires massive amounts of labeled data, and a nonlocal learning rule for changing the synapse strengths. This paper describes a learning algorithm that does not suffer from these two problems. It learns the weights of the lower layer of neural networks in a completely unsupervised fashion. The entire algorithm utilizes local learning rules which have conceptual biological plausibility. [Abstract] It is widely believed that end-to-end training with the backpropagation algorithm is essential for learning good feature detectors in early layers of artificial neural networks, so that these detectors are useful for the task performed by the higher layers of that neural network. At the same time, the traditional form of backpropagation is biologically implausible. In the present paper we propose an unusual learning rule, which has a degree of biological plausibility and which is motivated by Hebb’s idea that change of the synapse strength should be local—i.e., should depend only on the activities of the pre- and postsynaptic neurons. We design a learning algorithm that utilizes global inhibition in the hidden layer and is capable of learning early feature detectors in a completely unsupervised way. These learned lower-layer feature detectors can be used to train higher-layer weights in a usual supervised way so that the performance of the full network is comparable to the performance of standard feedforward networks trained end-to-end with a backpropagation algorithm on simple tasks.","eprint":"30926658","eprinttype":"pmid","keywords":"~INRMM-MiD:z-3D6W95X4,artificial-intelligence,artificial-neural-networks,back-propagation-networks,biological-deep-learning,computational-science,computational-science-automation,deep-learning,machine-learning,unsupervised-training","langid":"english","number":"16","bibtex":"@article{krotovUnsupervisedLearningCompeting2019,\n title = {Unsupervised Learning by Competing Hidden Units},\n author = {Krotov, Dmitry and Hopfield, John J.},\n date = {2019-04-16},\n journaltitle = {Proceedings of the National Academy of Sciences},\n shortjournal = {PNAS},\n volume = {116},\n pages = {7723--7731},\n issn = {0027-8424, 1091-6490},\n doi = {10.1073/pnas.1820458116},\n url = {https://doi.org/10.1073/pnas.1820458116},\n urldate = {2019-04-18},\n abstract = {[Significance]\nDespite great success of deep learning a question remains to what extent the computational properties of deep neural networks are similar to those of the human brain. The particularly nonbiological aspect of deep learning is the supervised training process with the backpropagation algorithm, which requires massive amounts of labeled data, and a nonlocal learning rule for changing the synapse strengths. This paper describes a learning algorithm that does not suffer from these two problems. It learns the weights of the lower layer of neural networks in a completely unsupervised fashion. The entire algorithm utilizes local learning rules which have conceptual biological plausibility.\n\n[Abstract]\nIt is widely believed that end-to-end training with the backpropagation algorithm is essential for learning good feature detectors in early layers of artificial neural networks, so that these detectors are useful for the task performed by the higher layers of that neural network. At the same time, the traditional form of backpropagation is biologically implausible. In the present paper we propose an unusual learning rule, which has a degree of biological plausibility and which is motivated by Hebb’s idea that change of the synapse strength should be local—i.e., should depend only on the activities of the pre- and postsynaptic neurons. We design a learning algorithm that utilizes global inhibition in the hidden layer and is capable of learning early feature detectors in a completely unsupervised way. These learned lower-layer feature detectors can be used to train higher-layer weights in a usual supervised way so that the performance of the full network is comparable to the performance of standard feedforward networks trained end-to-end with a backpropagation algorithm on simple tasks.},\n eprint = {30926658},\n eprinttype = {pmid},\n keywords = {~INRMM-MiD:z-3D6W95X4,artificial-intelligence,artificial-neural-networks,back-propagation-networks,biological-deep-learning,computational-science,computational-science-automation,deep-learning,machine-learning,unsupervised-training},\n langid = {english},\n number = {16}\n}\n\n","author_short":["Krotov, D.","Hopfield, J. J."],"key":"krotovUnsupervisedLearningCompeting2019","id":"krotovUnsupervisedLearningCompeting2019","bibbaseid":"krotov-hopfield-unsupervisedlearningbycompetinghiddenunits","role":"author","urls":{"Paper":"https://doi.org/10.1073/pnas.1820458116"},"keyword":["~INRMM-MiD:z-3D6W95X4","artificial-intelligence","artificial-neural-networks","back-propagation-networks","biological-deep-learning","computational-science","computational-science-automation","deep-learning","machine-learning","unsupervised-training"],"downloads":0},"bibtype":"article","biburl":"https://tmpfiles.org/dl/58794/INRMM.bib","creationDate":"2020-07-02T22:41:11.641Z","downloads":0,"keywords":["~inrmm-mid:z-3d6w95x4","artificial-intelligence","artificial-neural-networks","back-propagation-networks","biological-deep-learning","computational-science","computational-science-automation","deep-learning","machine-learning","unsupervised-training"],"search_terms":["unsupervised","learning","competing","hidden","units","krotov","hopfield"],"title":"Unsupervised Learning by Competing Hidden Units","year":null,"dataSources":["DXuKbcZTirdigFKPF"]}