Spatial Transformer for 3D Point Clouds

Spatial Transformer for 3D Point Clouds. Wang, J., Chakraborty, R., & Yu, S., X. IEEE Transactions on Pattern Analysis and Machine Intelligence, IEEE Computer Society, 6, 2019.

Paper

Spatial Transformer for 3D Point Clouds [link]

Website doi abstract bibtex

Deep neural networks are widely used for understanding 3D point clouds. At each point convolution layer, features are computed from local neighborhoods of 3D points and combined for subsequent processing in order to extract semantic information. Existing methods adopt the same individual point neighborhoods throughout the network layers, defined by the same metric on the fixed input point coordinates. This common practice is easy to implement but not necessarily optimal. Ideally, local neighborhoods should be different at different layers, as more latent information is extracted at deeper layers. We propose a novel end-to-end approach to learn different non-rigid transformations of the input point cloud so that optimal local neighborhoods can be adopted at each layer. We propose both linear (affine) and non-linear (projective and deformable) spatial transformers for 3D point clouds. With spatial transformers on the ShapeNet part segmentation dataset, the network achieves higher accuracy for all categories, with 8\% gain on earphones and rockets in particular. Our method also outperforms the state-of-the-art on other point cloud tasks such as classification, detection, and semantic segmentation. Visualizations show that spatial transformers can learn features more efficiently by dynamically altering local neighborhoods according to the geometry and semantics of 3D shapes in spite of their within-category variations. Our code is publicly available at https://github.com/samaonline/spatial-transformer-for-3d-point-clouds.

@article{
 title = {Spatial Transformer for 3D Point Clouds},
 type = {article},
 year = {2019},
 keywords = {3D detection,Convolution,Feature extraction,Measurement,Semantics,Shape,Task analysis,Three-dimensional displays,deformable,point cloud,segmentation,transformation},
 websites = {https://arxiv.org/abs/1906.10887v4},
 month = {6},
 publisher = {IEEE Computer Society},
 day = {26},
 id = {87b47b02-ebb4-3785-aca5-6a6df1d6adc1},
 created = {2021-09-15T11:24:33.381Z},
 accessed = {2021-09-15},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-12T08:29:57.242Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4,11276190-b8fe-4c3a-a42f-f604438ad4db},
 private_publication = {false},
 abstract = {Deep neural networks are widely used for understanding 3D point clouds. At
each point convolution layer, features are computed from local neighborhoods of
3D points and combined for subsequent processing in order to extract semantic
information. Existing methods adopt the same individual point neighborhoods
throughout the network layers, defined by the same metric on the fixed input
point coordinates. This common practice is easy to implement but not
necessarily optimal. Ideally, local neighborhoods should be different at
different layers, as more latent information is extracted at deeper layers. We
propose a novel end-to-end approach to learn different non-rigid
transformations of the input point cloud so that optimal local neighborhoods
can be adopted at each layer. We propose both linear (affine) and non-linear
(projective and deformable) spatial transformers for 3D point clouds. With
spatial transformers on the ShapeNet part segmentation dataset, the network
achieves higher accuracy for all categories, with 8\% gain on earphones and
rockets in particular. Our method also outperforms the state-of-the-art on
other point cloud tasks such as classification, detection, and semantic
segmentation. Visualizations show that spatial transformers can learn features
more efficiently by dynamically altering local neighborhoods according to the
geometry and semantics of 3D shapes in spite of their within-category
variations. Our code is publicly available at
https://github.com/samaonline/spatial-transformer-for-3d-point-clouds.},
 bibtype = {article},
 author = {Wang, Jiayun and Chakraborty, Rudrasis and Yu, Stella X.},
 doi = {10.1109/tpami.2021.3070341},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}
}

Downloads: 0

{"_id":"RTW2MqySzoez9ozFw","bibbaseid":"wang-chakraborty-yu-spatialtransformerfor3dpointclouds-2019","authorIDs":["rBXuGXyFAqasDAmkX"],"author_short":["Wang, J.","Chakraborty, R.","Yu, S., X."],"bibdata":{"title":"Spatial Transformer for 3D Point Clouds","type":"article","year":"2019","keywords":"3D detection,Convolution,Feature extraction,Measurement,Semantics,Shape,Task analysis,Three-dimensional displays,deformable,point cloud,segmentation,transformation","websites":"https://arxiv.org/abs/1906.10887v4","month":"6","publisher":"IEEE Computer Society","day":"26","id":"87b47b02-ebb4-3785-aca5-6a6df1d6adc1","created":"2021-09-15T11:24:33.381Z","accessed":"2021-09-15","file_attached":"true","profile_id":"48fc0258-023d-3602-860e-824092d62c56","group_id":"1ff583c0-be37-34fa-9c04-73c69437d354","last_modified":"2021-10-12T08:29:57.242Z","read":"true","starred":false,"authored":false,"confirmed":"true","hidden":false,"folder_uuids":"8d050117-e419-4b32-ad70-c875c74fa2b4,11276190-b8fe-4c3a-a42f-f604438ad4db","private_publication":false,"abstract":"Deep neural networks are widely used for understanding 3D point clouds. At\neach point convolution layer, features are computed from local neighborhoods of\n3D points and combined for subsequent processing in order to extract semantic\ninformation. Existing methods adopt the same individual point neighborhoods\nthroughout the network layers, defined by the same metric on the fixed input\npoint coordinates. This common practice is easy to implement but not\nnecessarily optimal. Ideally, local neighborhoods should be different at\ndifferent layers, as more latent information is extracted at deeper layers. We\npropose a novel end-to-end approach to learn different non-rigid\ntransformations of the input point cloud so that optimal local neighborhoods\ncan be adopted at each layer. We propose both linear (affine) and non-linear\n(projective and deformable) spatial transformers for 3D point clouds. With\nspatial transformers on the ShapeNet part segmentation dataset, the network\nachieves higher accuracy for all categories, with 8\\% gain on earphones and\nrockets in particular. Our method also outperforms the state-of-the-art on\nother point cloud tasks such as classification, detection, and semantic\nsegmentation. Visualizations show that spatial transformers can learn features\nmore efficiently by dynamically altering local neighborhoods according to the\ngeometry and semantics of 3D shapes in spite of their within-category\nvariations. Our code is publicly available at\nhttps://github.com/samaonline/spatial-transformer-for-3d-point-clouds.","bibtype":"article","author":"Wang, Jiayun and Chakraborty, Rudrasis and Yu, Stella X.","doi":"10.1109/tpami.2021.3070341","journal":"IEEE Transactions on Pattern Analysis and Machine Intelligence","bibtex":"@article{\n title = {Spatial Transformer for 3D Point Clouds},\n type = {article},\n year = {2019},\n keywords = {3D detection,Convolution,Feature extraction,Measurement,Semantics,Shape,Task analysis,Three-dimensional displays,deformable,point cloud,segmentation,transformation},\n websites = {https://arxiv.org/abs/1906.10887v4},\n month = {6},\n publisher = {IEEE Computer Society},\n day = {26},\n id = {87b47b02-ebb4-3785-aca5-6a6df1d6adc1},\n created = {2021-09-15T11:24:33.381Z},\n accessed = {2021-09-15},\n file_attached = {true},\n profile_id = {48fc0258-023d-3602-860e-824092d62c56},\n group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},\n last_modified = {2021-10-12T08:29:57.242Z},\n read = {true},\n starred = {false},\n authored = {false},\n confirmed = {true},\n hidden = {false},\n folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4,11276190-b8fe-4c3a-a42f-f604438ad4db},\n private_publication = {false},\n abstract = {Deep neural networks are widely used for understanding 3D point clouds. At\neach point convolution layer, features are computed from local neighborhoods of\n3D points and combined for subsequent processing in order to extract semantic\ninformation. Existing methods adopt the same individual point neighborhoods\nthroughout the network layers, defined by the same metric on the fixed input\npoint coordinates. This common practice is easy to implement but not\nnecessarily optimal. Ideally, local neighborhoods should be different at\ndifferent layers, as more latent information is extracted at deeper layers. We\npropose a novel end-to-end approach to learn different non-rigid\ntransformations of the input point cloud so that optimal local neighborhoods\ncan be adopted at each layer. We propose both linear (affine) and non-linear\n(projective and deformable) spatial transformers for 3D point clouds. With\nspatial transformers on the ShapeNet part segmentation dataset, the network\nachieves higher accuracy for all categories, with 8\\% gain on earphones and\nrockets in particular. Our method also outperforms the state-of-the-art on\nother point cloud tasks such as classification, detection, and semantic\nsegmentation. Visualizations show that spatial transformers can learn features\nmore efficiently by dynamically altering local neighborhoods according to the\ngeometry and semantics of 3D shapes in spite of their within-category\nvariations. Our code is publicly available at\nhttps://github.com/samaonline/spatial-transformer-for-3d-point-clouds.},\n bibtype = {article},\n author = {Wang, Jiayun and Chakraborty, Rudrasis and Yu, Stella X.},\n doi = {10.1109/tpami.2021.3070341},\n journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}\n}","author_short":["Wang, J.","Chakraborty, R.","Yu, S., X."],"urls":{"Paper":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c/file/3ff63dda-ad89-a175-2898-6872bde3697a/full_text.pdf.pdf","Website":"https://arxiv.org/abs/1906.10887v4"},"biburl":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c","bibbaseid":"wang-chakraborty-yu-spatialtransformerfor3dpointclouds-2019","role":"author","keyword":["3D detection","Convolution","Feature extraction","Measurement","Semantics","Shape","Task analysis","Three-dimensional displays","deformable","point cloud","segmentation","transformation"],"metadata":{"authorlinks":{"chakraborty, r":"https://rudra1988.github.io/publist.html"}},"downloads":0},"bibtype":"article","biburl":"https://bibbase.org/service/mendeley/bfbbf840-4c42-3914-a463-19024f50b30c","creationDate":"2021-01-02T03:13:29.148Z","downloads":0,"keywords":["3d detection","convolution","feature extraction","measurement","semantics","shape","task analysis","three-dimensional displays","deformable","point cloud","segmentation","transformation"],"search_terms":["spatial","transformer","point","clouds","wang","chakraborty","yu"],"title":"Spatial Transformer for 3D Point Clouds","year":2019,"dataSources":["vGP7QPCLoiQJeD5Ab","ya2CyA73rpZseyrZ8","2252seNhipfTmjEBQ"]}