@article{
 title = {Stream-Based Ground Segmentation for Real-Time LiDAR Point Cloud Processing on FPGA},
 type = {article},
 year = {2024},
 websites = {http://arxiv.org/abs/2408.10410},
 month = {8},
 day = {19},
 id = {e87d9bdc-04a2-3f14-bdbb-0a02c04d6a18},
 created = {2025-01-20T13:32:42.050Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:36:23.619Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {<u>1 cites</u>},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {This paper presents a novel and fast approach for ground plane segmentation in a LiDAR point cloud, specifically optimized for processing speed and hardware efficiency on FPGA hardware platforms. Our approach leverages a channel-based segmentation method with an advanced angular data repair technique and a cross-eight-way flood-fill algorithm. This innovative approach significantly reduces the number of iterations while ensuring the high accuracy of the segmented ground plane, which makes the stream-based hardware implementation possible. To validate the proposed approach, we conducted extensive experiments on the SemanticKITTI dataset. We introduced a bird's-eye view (BEV) evaluation metric tailored for the area representation of LiDAR segmentation tasks. Our method demonstrated superior performance in terms of BEV areas when compared to the existing approaches. Moreover, we presented an optimized hardware architecture targeted on a Zynq-7000 FPGA, compatible with LiDARs of various channel densities, i.e., 32, 64, and 128 channels. Our FPGA implementation operating at 160 MHz significantly outperforms the traditional computing platforms, which is 12 to 25 times faster than the CPU-based solutions and up to 6 times faster than the GPU-based solution, in addition to the benefit of low power consumption.},
 bibtype = {article},
 author = {Zhang, Xiao and Huang, Zhanhong and Antony, Garcia Gonzalez and Jachimczyk, Witek and Huang, Xinming},
 doi = {https://doi.org/10.48550/arXiv.2408.10410}
}

This paper presents a novel and fast approach for ground plane segmentation in a LiDAR point cloud, specifically optimized for processing speed and hardware efficiency on FPGA hardware platforms. Our approach leverages a channel-based segmentation method with an advanced angular data repair technique and a cross-eight-way flood-fill algorithm. This innovative approach significantly reduces the number of iterations while ensuring the high accuracy of the segmented ground plane, which makes the stream-based hardware implementation possible. To validate the proposed approach, we conducted extensive experiments on the SemanticKITTI dataset. We introduced a bird's-eye view (BEV) evaluation metric tailored for the area representation of LiDAR segmentation tasks. Our method demonstrated superior performance in terms of BEV areas when compared to the existing approaches. Moreover, we presented an optimized hardware architecture targeted on a Zynq-7000 FPGA, compatible with LiDARs of various channel densities, i.e., 32, 64, and 128 channels. Our FPGA implementation operating at 160 MHz significantly outperforms the traditional computing platforms, which is 12 to 25 times faster than the CPU-based solutions and up to 6 times faster than the GPU-based solution, in addition to the benefit of low power consumption.

@article{
 title = {An Integrated FPGA Accelerator for Deep Learning-Based 2D/3D Path Planning},
 type = {article},
 year = {2024},
 keywords = {FPGA,Path planning,PointNet,deep learning,neural path planning,point cloud processing},
 pages = {1442-1456},
 volume = {73},
 month = {6},
 publisher = {IEEE Computer Society},
 day = {1},
 id = {5fcfbc08-ae04-33ae-9e60-50d8e7646d85},
 created = {2025-04-07T08:22:45.822Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:21:01.992Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {5 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Path planning is a crucial component for realizing the autonomy of mobile robots. However, due to limited computational resources on mobile robots, it remains challenging to deploy state-of-the-art methods and achieve real-time performance. To address this, we propose P3Net (PointNet-based Path Planning Networks), a lightweight deep-learning-based method for 2D/3D path planning, and design an IP core (P3NetCore) targeting FPGA SoCs (Xilinx ZCU104). P3Net improves the algorithm and model architecture of the recently-proposed MPNet. P3Net employs an encoder with a PointNet backbone and a lightweight planning network in order to extract robust point cloud features and sample path points from a promising region. P3NetCore is comprised of the fully-pipelined point cloud encoder, batched bidirectional path planner, and parallel collision checker, to cover most part of the algorithm. On the 2D (3D) datasets, P3Net with the IP core runs 30.52-186.36x and 7.68-143.62x (15.69-93.26x and 5.30-45.27x) faster than ARM Cortex CPU and Nvidia Jetson while only consuming 0.255W (0.809W), and is up to 1278.14x (455.34x) power-efficient than the workstation. P3Net improves the success rate by up to 28.2% and plans a near-optimal path, leading to a significantly better tradeoff between computation and solution quality than MPNet and the state-of-the-art sampling-based methods.},
 bibtype = {article},
 author = {Sugiura, Keisuke and Matsutani, Hiroki},
 doi = {10.1109/TC.2024.3377895},
 journal = {IEEE Transactions on Computers},
 number = {6}
}

Path planning is a crucial component for realizing the autonomy of mobile robots. However, due to limited computational resources on mobile robots, it remains challenging to deploy state-of-the-art methods and achieve real-time performance. To address this, we propose P3Net (PointNet-based Path Planning Networks), a lightweight deep-learning-based method for 2D/3D path planning, and design an IP core (P3NetCore) targeting FPGA SoCs (Xilinx ZCU104). P3Net improves the algorithm and model architecture of the recently-proposed MPNet. P3Net employs an encoder with a PointNet backbone and a lightweight planning network in order to extract robust point cloud features and sample path points from a promising region. P3NetCore is comprised of the fully-pipelined point cloud encoder, batched bidirectional path planner, and parallel collision checker, to cover most part of the algorithm. On the 2D (3D) datasets, P3Net with the IP core runs 30.52-186.36x and 7.68-143.62x (15.69-93.26x and 5.30-45.27x) faster than ARM Cortex CPU and Nvidia Jetson while only consuming 0.255W (0.809W), and is up to 1278.14x (455.34x) power-efficient than the workstation. P3Net improves the success rate by up to 28.2% and plans a near-optimal path, leading to a significantly better tradeoff between computation and solution quality than MPNet and the state-of-the-art sampling-based methods.

@article{
 title = {CNN based plant disease identification using PYNQ FPGA},
 type = {article},
 year = {2024},
 keywords = {CNN,FPGA,Image Processing,PYNQ,Plant Disease},
 volume = {6},
 month = {12},
 publisher = {Academic Press},
 day = {1},
 id = {a0c12443-d995-3f57-81aa-8e5f2330de5e},
 created = {2025-04-07T10:37:06.111Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:21:31.542Z},
 read = {false},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {9 cites<br/><br/><b>Paper description</b><br/>CNN-based plant disease identification using PYNQ. FPGA<br/>parallel processing to enhance efficiency.<br/><br/><b>Methods</b>: DCNN for FPGA and transfer learning with ResNet34.<br/><br/><b>Tools</b>: Xilinx PYNQ, Python<br/><br/><b>Materials</b>: Kaggle’s Plant Village dataset<br/><br/><b>Results</b><br/>- Fine-tuning improved accuracy by 2.8% and F1 score by 3.1%.},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {This research presents a novel approach for plant disease identification utilizing Convolutional Neural Networks (CNNs) and the PYNQ FPGA platform. The study leverages the parallel processing capabilities of FPGAs to accelerate CNN inference, aiming to enhance the efficiency of plant disease detection in agricultural settings. The implementation involves optimizing the CNN architecture for deployment on the PYNQ FPGA, considering factors such as image size and learning rates. Through experimentation, the research refines hyper parameters, achieving improved accuracy and F1 scores. Visualizations using heat maps highlight the CNN's reliance on color, shape, and texture for feature extraction in disease identification. The integration of FPGA technology demonstrates promising advancements in real-time, high-performance plant disease classification, offering potential benefits for precision agriculture and crop management. This research contributes to the growing field of FPGA-accelerated deep learning applications in agro technology, addressing challenges in plant health monitoring and fostering sustainable agricultural practices.},
 bibtype = {article},
 author = {Perumal, Vivek Karthick and T, Supriyaa and P R, Santhosh and S, Dhanasekaran},
 doi = {10.1016/j.sasc.2024.200088},
 journal = {Systems and Soft Computing}
}

This research presents a novel approach for plant disease identification utilizing Convolutional Neural Networks (CNNs) and the PYNQ FPGA platform. The study leverages the parallel processing capabilities of FPGAs to accelerate CNN inference, aiming to enhance the efficiency of plant disease detection in agricultural settings. The implementation involves optimizing the CNN architecture for deployment on the PYNQ FPGA, considering factors such as image size and learning rates. Through experimentation, the research refines hyper parameters, achieving improved accuracy and F1 scores. Visualizations using heat maps highlight the CNN's reliance on color, shape, and texture for feature extraction in disease identification. The integration of FPGA technology demonstrates promising advancements in real-time, high-performance plant disease classification, offering potential benefits for precision agriculture and crop management. This research contributes to the growing field of FPGA-accelerated deep learning applications in agro technology, addressing challenges in plant health monitoring and fostering sustainable agricultural practices.

@article{
 title = {Fast evaluation of real spherical harmonics and their derivatives in Cartesian coordinates},
 type = {article},
 year = {2023},
 pages = {1-8},
 websites = {http://arxiv.org/abs/2302.08381},
 id = {fc9fc073-ca3e-3cb3-ac41-61f2af9c079e},
 created = {2023-05-03T13:16:39.666Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.693Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Bigi2023},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Spherical harmonics provide a smooth, orthogonal, and symmetry-adapted basis to expand functions on a sphere, and they are used routinely in computer graphics, signal processing and different fields of science, from geology to quantum chemistry. More recently, spherical harmonics have become a key component of rotationally equivariant models for geometric deep learning, where they are used in combination with distance-dependent functions to describe the distribution of neighbors within local spherical environments within a point cloud. We present a fast and elegant algorithm for the evaluation of the real-valued spherical harmonics. Our construction integrates many of the desirable features of existing schemes and allows to compute Cartesian derivatives in a numerically stable and computationally efficient manner. We provide an efficient C implementation of the proposed algorithm, along with easy-to-use Python bindings.},
 bibtype = {article},
 author = {Bigi, Filippo and Ceriotti, Michele}
}

Spherical harmonics provide a smooth, orthogonal, and symmetry-adapted basis to expand functions on a sphere, and they are used routinely in computer graphics, signal processing and different fields of science, from geology to quantum chemistry. More recently, spherical harmonics have become a key component of rotationally equivariant models for geometric deep learning, where they are used in combination with distance-dependent functions to describe the distribution of neighbors within local spherical environments within a point cloud. We present a fast and elegant algorithm for the evaluation of the real-valued spherical harmonics. Our construction integrates many of the desirable features of existing schemes and allows to compute Cartesian derivatives in a numerically stable and computationally efficient manner. We provide an efficient C implementation of the proposed algorithm, along with easy-to-use Python bindings.

@article{
 title = {Reducing SO(3) Convolutions to SO(2) for Efficient Equivariant GNNs},
 type = {article},
 year = {2023},
 websites = {http://arxiv.org/abs/2302.03655},
 id = {edcf5002-2e33-347a-bdc8-380683d5a8b0},
 created = {2023-05-03T13:16:40.626Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.626Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Passaro2023},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Graph neural networks that model 3D data, such as point clouds or atoms, are typically desired to be $SO(3)$ equivariant, i.e., equivariant to 3D rotations. Unfortunately equivariant convolutions, which are a fundamental operation for equivariant networks, increase significantly in computational complexity as higher-order tensors are used. In this paper, we address this issue by reducing the $SO(3)$ convolutions or tensor products to mathematically equivalent convolutions in $SO(2)$ . This is accomplished by aligning the node embeddings' primary axis with the edge vectors, which sparsifies the tensor product and reduces the computational complexity from $O(L^6)$ to $O(L^3)$, where $L$ is the degree of the representation. We demonstrate the potential implications of this improvement by proposing the Equivariant Spherical Channel Network (eSCN), a graph neural network utilizing our novel approach to equivariant convolutions, which achieves state-of-the-art results on the large-scale OC-20 dataset.},
 bibtype = {article},
 author = {Passaro, Saro and Zitnick, C. Lawrence},
 number = {3}
}

Graph neural networks that model 3D data, such as point clouds or atoms, are typically desired to be $SO(3)$ equivariant, i.e., equivariant to 3D rotations. Unfortunately equivariant convolutions, which are a fundamental operation for equivariant networks, increase significantly in computational complexity as higher-order tensors are used. In this paper, we address this issue by reducing the $SO(3)$ convolutions or tensor products to mathematically equivalent convolutions in $SO(2)$ . This is accomplished by aligning the node embeddings' primary axis with the edge vectors, which sparsifies the tensor product and reduces the computational complexity from $O(L^6)$ to $O(L^3)$, where $L$ is the degree of the representation. We demonstrate the potential implications of this improvement by proposing the Equivariant Spherical Channel Network (eSCN), a graph neural network utilizing our novel approach to equivariant convolutions, which achieves state-of-the-art results on the large-scale OC-20 dataset.

@article{
 title = {3D Spectral Domain Registration-Based Visual Servoing},
 type = {article},
 year = {2023},
 websites = {http://arxiv.org/abs/2303.15857},
 id = {c52d0131-474c-337a-8f32-b5cf4c113f79},
 created = {2023-05-03T13:16:40.844Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.388Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Adjigble2023},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {This paper presents a spectral domain registration-based visual servoing scheme that works on 3D point clouds. Specifically, we propose a 3D model/point cloud alignment method, which works by finding a global transformation between reference and target point clouds using spectral analysis. A 3D Fast Fourier Transform (FFT) in R3 is used for the translation estimation, and the real spherical harmonics in SO(3) are used for the rotations estimation. Such an approach allows us to derive a decoupled 6 degrees of freedom (DoF) controller, where we use gradient ascent optimisation to minimise translation and rotational costs. We then show how this methodology can be used to regulate a robot arm to perform a positioning task. In contrast to the existing state-of-the-art depth-based visual servoing methods that either require dense depth maps or dense point clouds, our method works well with partial point clouds and can effectively handle larger transformations between the reference and the target positions. Furthermore, the use of spectral data (instead of spatial data) for transformation estimation makes our method robust to sensor-induced noise and partial occlusions. We validate our approach by performing experiments using point clouds acquired by a robot-mounted depth camera. Obtained results demonstrate the effectiveness of our visual servoing approach.},
 bibtype = {article},
 author = {Adjigble, Maxime and Tamadazte, Brahim and de Farias, Cristiana and Stolkin, Rustam and Marturi, Naresh}
}

This paper presents a spectral domain registration-based visual servoing scheme that works on 3D point clouds. Specifically, we propose a 3D model/point cloud alignment method, which works by finding a global transformation between reference and target point clouds using spectral analysis. A 3D Fast Fourier Transform (FFT) in R3 is used for the translation estimation, and the real spherical harmonics in SO(3) are used for the rotations estimation. Such an approach allows us to derive a decoupled 6 degrees of freedom (DoF) controller, where we use gradient ascent optimisation to minimise translation and rotational costs. We then show how this methodology can be used to regulate a robot arm to perform a positioning task. In contrast to the existing state-of-the-art depth-based visual servoing methods that either require dense depth maps or dense point clouds, our method works well with partial point clouds and can effectively handle larger transformations between the reference and the target positions. Furthermore, the use of spectral data (instead of spatial data) for transformation estimation makes our method robust to sensor-induced noise and partial occlusions. We validate our approach by performing experiments using point clouds acquired by a robot-mounted depth camera. Obtained results demonstrate the effectiveness of our visual servoing approach.

@article{
 title = {Evaluate Geometry of Radiance Field with Low-frequency Color Prior},
 type = {article},
 year = {2023},
 websites = {http://arxiv.org/abs/2304.04351},
 id = {2993599e-9859-3b83-8936-d71f252f2aa8},
 created = {2023-05-03T13:16:40.951Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.361Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Fang2023},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Radiance field is an effective representation of 3D scenes, which has been widely adopted in novel-view synthesis and 3D reconstruction. It is still an open and challenging problem to evaluate the geometry, i.e., the density field, as the ground-truth is almost impossible to be obtained. One alternative indirect solution is to transform the density field into a point-cloud and compute its Chamfer Distance with the scanned ground-truth. However, many widely-used datasets have no point-cloud ground-truth since the scanning process along with the equipment is expensive and complicated. To this end, we propose a novel metric, named Inverse Mean Residual Color (IMRC), which can evaluate the geometry only with the observation images. Our key insight is that the better the geometry is, the lower-frequency the computed color field is. From this insight, given reconstructed density field and the observation images, we design a closed-form method to approximate the color field with low-frequency spherical harmonics and compute the inverse mean residual color. Then the higher the IMRC, the better the geometry. Qualitative and quantitative experimental results verify the effectiveness of our proposed IMRC metric. We also benchmark several state-of-the-art methods using IMRC to promote future related research.},
 bibtype = {article},
 author = {Fang, Qihang and Song, Yafei and Li, Keqiang and Shen, Li and Wu, Huaiyu and Xiong, Gang and Bo, Liefeng}
}

Radiance field is an effective representation of 3D scenes, which has been widely adopted in novel-view synthesis and 3D reconstruction. It is still an open and challenging problem to evaluate the geometry, i.e., the density field, as the ground-truth is almost impossible to be obtained. One alternative indirect solution is to transform the density field into a point-cloud and compute its Chamfer Distance with the scanned ground-truth. However, many widely-used datasets have no point-cloud ground-truth since the scanning process along with the equipment is expensive and complicated. To this end, we propose a novel metric, named Inverse Mean Residual Color (IMRC), which can evaluate the geometry only with the observation images. Our key insight is that the better the geometry is, the lower-frequency the computed color field is. From this insight, given reconstructed density field and the observation images, we design a closed-form method to approximate the color field with low-frequency spherical harmonics and compute the inverse mean residual color. Then the higher the IMRC, the better the geometry. Qualitative and quantitative experimental results verify the effectiveness of our proposed IMRC metric. We also benchmark several state-of-the-art methods using IMRC to promote future related research.

@article{
 title = {MACARONS: Mapping And Coverage Anticipation with RGB Online Self-Supervision},
 type = {article},
 year = {2023},
 pages = {940-951},
 websites = {http://arxiv.org/abs/2303.03315},
 id = {3fbfd8cc-0cb7-3712-830a-5a9b02b198f0},
 created = {2023-06-22T10:06:22.408Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-06-22T10:06:44.120Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {cbcad584-0c50-48fe-a0d7-5b4c781cba83},
 private_publication = {false},
 abstract = {We introduce a method that simultaneously learns to explore new large environments and to reconstruct them in 3D from color images only. This is closely related to the Next Best View problem (NBV), where one has to identify where to move the camera next to improve the coverage of an unknown scene. However, most of the current NBV methods rely on depth sensors, need 3D supervision and/or do not scale to large scenes. Our method requires only a color camera and no 3D supervision. It simultaneously learns in a self-supervised fashion to predict a "volume occupancy field" from color images and, from this field, to predict the NBV. Thanks to this approach, our method performs well on new scenes as it is not biased towards any training 3D data. We demonstrate this on a recent dataset made of various 3D scenes and show it performs even better than recent methods requiring a depth sensor, which is not a realistic assumption for outdoor scenes captured with a flying drone.},
 bibtype = {article},
 author = {Guédon, Antoine and Monnier, Tom and Monasse, Pascal and Lepetit, Vincent}
}

We introduce a method that simultaneously learns to explore new large environments and to reconstruct them in 3D from color images only. This is closely related to the Next Best View problem (NBV), where one has to identify where to move the camera next to improve the coverage of an unknown scene. However, most of the current NBV methods rely on depth sensors, need 3D supervision and/or do not scale to large scenes. Our method requires only a color camera and no 3D supervision. It simultaneously learns in a self-supervised fashion to predict a "volume occupancy field" from color images and, from this field, to predict the NBV. Thanks to this approach, our method performs well on new scenes as it is not biased towards any training 3D data. We demonstrate this on a recent dataset made of various 3D scenes and show it performs even better than recent methods requiring a depth sensor, which is not a realistic assumption for outdoor scenes captured with a flying drone.

@article{
 title = {Change detection of urban objects using 3D point clouds: A review},
 type = {article},
 year = {2023},
 keywords = {Applications,Change detection,Point clouds,Urban objects},
 pages = {228-255},
 volume = {197},
 websites = {https://doi.org/10.1016/j.isprsjprs.2023.01.010},
 publisher = {Elsevier B.V.},
 id = {918a225a-8193-3641-872c-67f3fb8feac9},
 created = {2023-06-22T10:06:22.411Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-06-22T10:06:55.898Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {cbcad584-0c50-48fe-a0d7-5b4c781cba83},
 private_publication = {false},
 abstract = {Over recent decades, 3D point clouds have been a popular data source applied in automatic change detection in a wide variety of applications. Compared with 2D images, using 3D point clouds for change detection can provide an alternative solution offering different modalities and enabling a highly detailed 3D geometric and attribute analysis. This article provides a comprehensive review of point-cloud-based 3D change detection for urban objects. Specifically, in this study, we had two primary aims: (i) to ascertain the critical techniques in change detection, as well as their strengths and weaknesses, including data registration, variance estimation, and change analysis; (ii) to contextualize the up-to-date uses of point clouds in change detection and to explore representative applications of land cover and land use monitoring, vegetation surveys, construction automation, building and indoor investigations, and traffic and transportation monitoring. A workflow following the PRISMA 2020 rules was applied for the search and selection of reviewed articles, with a brief statistical analysis of the selected articles. Additionally, we examined the limitations of current change detection technology and discussed current research gaps between state-of-the-art techniques and engineering demands. Several remaining issues, such as the reliability of datasets, uncertainty in results, and contribution of semantics in change detection, have been identified and discussed. Ultimately, this review sheds light on prospective research directions to meet the urgent needs of anticipated applications.},
 bibtype = {article},
 author = {Stilla, Uwe and Xu, Yusheng},
 doi = {10.1016/j.isprsjprs.2023.01.010},
 journal = {ISPRS Journal of Photogrammetry and Remote Sensing},
 number = {February}
}

Over recent decades, 3D point clouds have been a popular data source applied in automatic change detection in a wide variety of applications. Compared with 2D images, using 3D point clouds for change detection can provide an alternative solution offering different modalities and enabling a highly detailed 3D geometric and attribute analysis. This article provides a comprehensive review of point-cloud-based 3D change detection for urban objects. Specifically, in this study, we had two primary aims: (i) to ascertain the critical techniques in change detection, as well as their strengths and weaknesses, including data registration, variance estimation, and change analysis; (ii) to contextualize the up-to-date uses of point clouds in change detection and to explore representative applications of land cover and land use monitoring, vegetation surveys, construction automation, building and indoor investigations, and traffic and transportation monitoring. A workflow following the PRISMA 2020 rules was applied for the search and selection of reviewed articles, with a brief statistical analysis of the selected articles. Additionally, we examined the limitations of current change detection technology and discussed current research gaps between state-of-the-art techniques and engineering demands. Several remaining issues, such as the reliability of datasets, uncertainty in results, and contribution of semantics in change detection, have been identified and discussed. Ultimately, this review sheds light on prospective research directions to meet the urgent needs of anticipated applications.

@article{
 title = {Make-It-3D: High-Fidelity 3D Creation from A Single Image with Diffusion Prior},
 type = {article},
 year = {2023},
 websites = {http://arxiv.org/abs/2303.14184},
 id = {b38659d8-07cd-3bee-a402-4775596c3600},
 created = {2023-06-22T10:06:22.417Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-06-22T10:06:50.529Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {cbcad584-0c50-48fe-a0d7-5b4c781cba83},
 private_publication = {false},
 abstract = {In this work, we investigate the problem of creating high-fidelity 3D content from only a single image. This is inherently challenging: it essentially involves estimating the underlying 3D geometry while simultaneously hallucinating unseen textures. To address this challenge, we leverage prior knowledge from a well-trained 2D diffusion model to act as 3D-aware supervision for 3D creation. Our approach, Make-It-3D, employs a two-stage optimization pipeline: the first stage optimizes a neural radiance field by incorporating constraints from the reference image at the frontal view and diffusion prior at novel views; the second stage transforms the coarse model into textured point clouds and further elevates the realism with diffusion prior while leveraging the high-quality textures from the reference image. Extensive experiments demonstrate that our method outperforms prior works by a large margin, resulting in faithful reconstructions and impressive visual quality. Our method presents the first attempt to achieve high-quality 3D creation from a single image for general objects and enables various applications such as text-to-3D creation and texture editing.},
 bibtype = {article},
 author = {Tang, Junshu and Wang, Tengfei and Zhang, Bo and Zhang, Ting and Yi, Ran and Ma, Lizhuang and Chen, Dong}
}

In this work, we investigate the problem of creating high-fidelity 3D content from only a single image. This is inherently challenging: it essentially involves estimating the underlying 3D geometry while simultaneously hallucinating unseen textures. To address this challenge, we leverage prior knowledge from a well-trained 2D diffusion model to act as 3D-aware supervision for 3D creation. Our approach, Make-It-3D, employs a two-stage optimization pipeline: the first stage optimizes a neural radiance field by incorporating constraints from the reference image at the frontal view and diffusion prior at novel views; the second stage transforms the coarse model into textured point clouds and further elevates the realism with diffusion prior while leveraging the high-quality textures from the reference image. Extensive experiments demonstrate that our method outperforms prior works by a large margin, resulting in faithful reconstructions and impressive visual quality. Our method presents the first attempt to achieve high-quality 3D creation from a single image for general objects and enables various applications such as text-to-3D creation and texture editing.

@article{
 title = {A Rapid Water Region Reconstruction Scheme in 3D Watershed Scene Generated by UAV Oblique Photography},
 type = {article},
 year = {2023},
 keywords = {3D reconstruction,oblique photography,real-scene 3D environment,twin watershed,water region},
 pages = {1-19},
 volume = {15},
 id = {d12bc583-1847-3672-a356-a2443ee891e3},
 created = {2023-06-22T10:06:22.562Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-06-22T10:07:04.855Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {cbcad584-0c50-48fe-a0d7-5b4c781cba83},
 private_publication = {false},
 abstract = {Oblique photography technology based on UAV (unmanned aerial vehicle) provides an effective means for the rapid, real-scene 3D reconstruction of geographical objects on a watershed scale. However, existing research cannot achieve the automatic and high-precision reconstruction of water regions due to the sensitivity of water surface patterns to wind and waves, reflections of objects on the shore, etc. To solve this problem, a novel rapid reconstruction scheme for water regions in 3D models of oblique photography is proposed in this paper. It extracts the boundaries of water regions firstly using a designed eight-neighborhood traversal algorithm, and then reconstructs the triangulated irregular network (TIN) of water regions. Afterwards, the corresponding texture images of water regions are intelligently selected and processed using a designed method based on coordinate matching, image stitching and clipping. Finally, the processed texture images are mapped to the obtained TIN, and the real information about water regions can be reconstructed, visualized and integrated into the original real-scene 3D environment. Experimental results have shown that the proposed scheme can rapidly and accurately reconstruct water regions in 3D models of oblique photography. The outcome of this work can refine the current technical system of 3D modeling by UAV oblique photography and expand its application in the construction of twin watershed, twin city, etc.},
 bibtype = {article},
 author = {Qiu, Yinguo and Jiao, Yaqin and Luo, Juhua and Tan, Zhenyu and Huang, Linsheng and Zhao, Jinling and Xiao, Qitao and Duan, Hongtao},
 doi = {10.3390/rs15051211},
 journal = {Remote Sensing},
 number = {5}
}

Oblique photography technology based on UAV (unmanned aerial vehicle) provides an effective means for the rapid, real-scene 3D reconstruction of geographical objects on a watershed scale. However, existing research cannot achieve the automatic and high-precision reconstruction of water regions due to the sensitivity of water surface patterns to wind and waves, reflections of objects on the shore, etc. To solve this problem, a novel rapid reconstruction scheme for water regions in 3D models of oblique photography is proposed in this paper. It extracts the boundaries of water regions firstly using a designed eight-neighborhood traversal algorithm, and then reconstructs the triangulated irregular network (TIN) of water regions. Afterwards, the corresponding texture images of water regions are intelligently selected and processed using a designed method based on coordinate matching, image stitching and clipping. Finally, the processed texture images are mapped to the obtained TIN, and the real information about water regions can be reconstructed, visualized and integrated into the original real-scene 3D environment. Experimental results have shown that the proposed scheme can rapidly and accurately reconstruct water regions in 3D models of oblique photography. The outcome of this work can refine the current technical system of 3D modeling by UAV oblique photography and expand its application in the construction of twin watershed, twin city, etc.

@inproceedings{
 title = {Convolutional Neural Networks on the Edge: A Comparison Between FPGA and GPU},
 type = {inproceedings},
 year = {2023},
 month = {6},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {26},
 city = {Shanghai, China},
 id = {e0fd97ea-4e2b-36c4-9f56-0fb0e3190e79},
 created = {2023-11-07T10:04:43.974Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:40:19.680Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {1 cite},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {With more computation tasks being pushed towards the edge, it becomes increasingly challenging to decide the right hardware approach for running edge AI tasks. Besides customized AI accelerators, existing hardware platforms such as FPGAs have been employed to support edge computing. Recently, small GPUs have also been developed to run AI tasks that are tailored for edge. While each platform exhibits its own advantages, the shortcomings are also identified. In this work, we present a study where a head to head comparison is conducted by running the same CNN on two popular edge platforms, FPGAs and GPUs. We compare multiple dimensions such as power, inference speed, ease of development and accuracy. The comparison results can provide an initial guidance for edge computing developers who don't have immediate access to both platforms.},
 bibtype = {inproceedings},
 author = {Wei, Yichen and Gong, Siyi and Mei, Hongfei and Shi, Longxing and Guo, Xinfei},
 doi = {10.1109/CSTIC58779.2023.10219304},
 booktitle = {2023 China Semiconductor Technology International Conference, CSTIC 2023}
}

With more computation tasks being pushed towards the edge, it becomes increasingly challenging to decide the right hardware approach for running edge AI tasks. Besides customized AI accelerators, existing hardware platforms such as FPGAs have been employed to support edge computing. Recently, small GPUs have also been developed to run AI tasks that are tailored for edge. While each platform exhibits its own advantages, the shortcomings are also identified. In this work, we present a study where a head to head comparison is conducted by running the same CNN on two popular edge platforms, FPGAs and GPUs. We compare multiple dimensions such as power, inference speed, ease of development and accuracy. The comparison results can provide an initial guidance for edge computing developers who don't have immediate access to both platforms.

@article{
 title = {Benchmarking edge computing devices for grape bunches and trunks detection using accelerated object detection single shot multibox deep learning models},
 type = {article},
 year = {2023},
 keywords = {Embedded systems,Heterogeneous platforms,Object detection,RetinaNet resNet,SSD resNet},
 volume = {117},
 month = {1},
 publisher = {Elsevier Ltd},
 day = {1},
 id = {a252719c-4909-323a-bc20-37001c166cf5},
 created = {2023-11-16T13:12:41.041Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:21:09.715Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {15 cites<br/><br/>Benchmarking on GPU FPGA devices<br/>Tested on Vine grapes images - measure FPS<br/><br/>Materials Methods:<br/>DL was built and trained in <b>TensorFlow 2.8 Keras</b>.<br/><b>TF-TRT</b> NVIDIA library that operates with TensorFlow and TensorRT (TRT)<br/><b>Vitis-AI</b>: quantised model INT8 weights and converted to a readable <b>DPU format</b><br/><br/><b>RetinaNet ResNet-50</b> archs used},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Purpose:
Visual perception enables robots to perceive the environment. Visual data is processed using computer vision algorithms that are usually time-expensive and require powerful devices to process the visual data in real-time, which is unfeasible for open-field robots with limited energy. This work benchmarks the performance of different heterogeneous platforms for object detection in real-time. This research benchmarks three architectures: embedded GPU—Graphical Processing Units (such as NVIDIA Jetson Nano 2 GB and 4 GB, and NVIDIA Jetson TX2), TPU—Tensor Processing Unit (such as Coral Dev Board TPU), and DPU—Deep Learning Processor Unit (such as in AMD/Xilinx ZCU104 Development Board, and AMD/Xilinx Kria KV260 Starter Kit).
Methods:
The authors used the RetinaNet ResNet-50 fine-tuned using the natural VineSet dataset. After the trained model was converted and compiled for target-specific hardware formats to improve the execution efficiency.
Conclusions and Results:
The platforms were assessed in terms of performance of the evaluation metrics and efficiency (time of inference). Graphical Processing Units (GPUs) were the slowest devices, running at 3 FPS to 5 FPS, and Field Programmable Gate Arrays (FPGAs) were the fastest devices, running at 14 FPS to 25 FPS. The efficiency of the Tensor Processing Unit (TPU) is irrelevant and similar to NVIDIA Jetson TX2. TPU and GPU are the most power-efficient, consuming about 5 W. The performance differences, in the evaluation metrics, across devices are irrelevant and have an F1 of about 70 % and mean Average Precision (mAP) of about 60%.},
 bibtype = {article},
 author = {Magalhães, Sandro Costa and dos Santos, Filipe Neves and Machado, Pedro and Moreira, António Paulo and Dias, Jorge},
 doi = {10.1016/j.engappai.2022.105604},
 journal = {Engineering Applications of Artificial Intelligence}
}

Purpose: Visual perception enables robots to perceive the environment. Visual data is processed using computer vision algorithms that are usually time-expensive and require powerful devices to process the visual data in real-time, which is unfeasible for open-field robots with limited energy. This work benchmarks the performance of different heterogeneous platforms for object detection in real-time. This research benchmarks three architectures: embedded GPU—Graphical Processing Units (such as NVIDIA Jetson Nano 2 GB and 4 GB, and NVIDIA Jetson TX2), TPU—Tensor Processing Unit (such as Coral Dev Board TPU), and DPU—Deep Learning Processor Unit (such as in AMD/Xilinx ZCU104 Development Board, and AMD/Xilinx Kria KV260 Starter Kit). Methods: The authors used the RetinaNet ResNet-50 fine-tuned using the natural VineSet dataset. After the trained model was converted and compiled for target-specific hardware formats to improve the execution efficiency. Conclusions and Results: The platforms were assessed in terms of performance of the evaluation metrics and efficiency (time of inference). Graphical Processing Units (GPUs) were the slowest devices, running at 3 FPS to 5 FPS, and Field Programmable Gate Arrays (FPGAs) were the fastest devices, running at 14 FPS to 25 FPS. The efficiency of the Tensor Processing Unit (TPU) is irrelevant and similar to NVIDIA Jetson TX2. TPU and GPU are the most power-efficient, consuming about 5 W. The performance differences, in the evaluation metrics, across devices are irrelevant and have an F1 of about 70 % and mean Average Precision (mAP) of about 60%.

@article{
 title = {FPGA–accelerated CNN for real-time plant disease identification},
 type = {article},
 year = {2023},
 keywords = {CNN,Edge computation,FPGA–accelerated,Plant disease identification,Real-time},
 volume = {207},
 month = {4},
 publisher = {Elsevier B.V.},
 day = {1},
 id = {02aebeb4-86f2-3f59-9bf4-7454e45b78fe},
 created = {2025-01-20T12:09:23.444Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:23:17.021Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {43 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Using convolutional neural network (CNN) to identify plant diseases in-situ is a hot research topic in smart agriculture. Due to the memory-intensive and compute-intensive characteristics of CNN algorithm, it is difficult to implement CNN on edge terminals with limited memory and computational resources. In this paper, Field Programmable Gate Array (FPGA) is used to accelerate CNN to identify plant diseases. First, a 7-layer simple-structured network called “LiteCNN”, with only 176 K parameters and 78.47 M floating point operations (FLOPs) was designed. And knowledge distillation method was used to train LiteCNN, making that the accuracy reaches 95.24 %. Secondly, the acceleration circuit of LiteCNN was designed and implemented on “ZYNQ Z7-Lite 7020″ FPGA board. To compress the network and speed up plant disease identification, the following methods were applied: 1) Separable convolution took place of regular convolution, and a low-redundancy block convolution approach was used to load data; 2) The Batch Normalization (BN) layer was fused into the previous convolutional layer (or fully-connected layer); 3) Feature data and model parameters were expressed by half float data. As the basic function of the circuit achieved, methods including unrolling the for-loop, pipelining the for-loop, loop flattening and array partitioning were used to optimize the parallelism of the circuit. Finally, LiteCNN on the FPGA board was verified. The plant disease identification accuracy was 95.71 %, the inference speed was 0.071 s per frame, and the power consumption was 2.41 W. The results show that this paper proposed a low-power, high-accuracy and fast-speed plant disease identification terminal, which can be well applied for real-time plant disease identification in the field.},
 bibtype = {article},
 author = {Luo, Yuexuan and Cai, Xiang and Qi, Jiandong and Guo, Dongdong and Che, Wenqing},
 doi = {10.1016/j.compag.2023.107715},
 journal = {Computers and Electronics in Agriculture}
}

Using convolutional neural network (CNN) to identify plant diseases in-situ is a hot research topic in smart agriculture. Due to the memory-intensive and compute-intensive characteristics of CNN algorithm, it is difficult to implement CNN on edge terminals with limited memory and computational resources. In this paper, Field Programmable Gate Array (FPGA) is used to accelerate CNN to identify plant diseases. First, a 7-layer simple-structured network called “LiteCNN”, with only 176 K parameters and 78.47 M floating point operations (FLOPs) was designed. And knowledge distillation method was used to train LiteCNN, making that the accuracy reaches 95.24 %. Secondly, the acceleration circuit of LiteCNN was designed and implemented on “ZYNQ Z7-Lite 7020″ FPGA board. To compress the network and speed up plant disease identification, the following methods were applied: 1) Separable convolution took place of regular convolution, and a low-redundancy block convolution approach was used to load data; 2) The Batch Normalization (BN) layer was fused into the previous convolutional layer (or fully-connected layer); 3) Feature data and model parameters were expressed by half float data. As the basic function of the circuit achieved, methods including unrolling the for-loop, pipelining the for-loop, loop flattening and array partitioning were used to optimize the parallelism of the circuit. Finally, LiteCNN on the FPGA board was verified. The plant disease identification accuracy was 95.71 %, the inference speed was 0.071 s per frame, and the power consumption was 2.41 W. The results show that this paper proposed a low-power, high-accuracy and fast-speed plant disease identification terminal, which can be well applied for real-time plant disease identification in the field.

@inproceedings{
 title = {FPGA-based Acceleration of Lidar Point Cloud Processing and Detection on the Edge},
 type = {inproceedings},
 year = {2023},
 keywords = {FPGA,ITS-S,Lidar,Object Detection,Point Cloud Processing},
 volume = {2023-June},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {b3b0590c-6559-3726-aa8a-681cfc51583e},
 created = {2025-01-20T13:01:35.429Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:35:13.630Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {3 cites},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Edge nodes such as Intelligent Transportation System Stations are becoming increasingly relevant in the context of automated driving as they provide connected vehicles with additional information to support their automated driving functions. However, the power budget for these edge nodes is limited and data has to be processed in real-time to be of use to automated driving functions. In this work, we present a system for processing raw lidar data in real-time on an FPGA, resulting in a significant reduction in power consumption compared to conventional hardware. Our approach leads to a 42.4% reduction in power consumption while maintaining the quality of the results. Processing two 128-layer surround-view lidar point clouds takes 522 ms per frame and an average power consumption of 39.3 W for the CPU and 34.5W for the FPGA. Our optimizations surpass the state-of-the-art by up to 193 times.},
 bibtype = {inproceedings},
 author = {Latotzke, Cecilia and Kloeker, Amarin and Schoening, Simon and Kemper, Fabian and Slimi, Mazen and Eckstein, Lutz and Gemmeke, Tobias},
 doi = {10.1109/IV55152.2023.10186612},
 booktitle = {IEEE Intelligent Vehicles Symposium, Proceedings}
}

Edge nodes such as Intelligent Transportation System Stations are becoming increasingly relevant in the context of automated driving as they provide connected vehicles with additional information to support their automated driving functions. However, the power budget for these edge nodes is limited and data has to be processed in real-time to be of use to automated driving functions. In this work, we present a system for processing raw lidar data in real-time on an FPGA, resulting in a significant reduction in power consumption compared to conventional hardware. Our approach leads to a 42.4% reduction in power consumption while maintaining the quality of the results. Processing two 128-layer surround-view lidar point clouds takes 522 ms per frame and an average power consumption of 39.3 W for the CPU and 34.5W for the FPGA. Our optimizations surpass the state-of-the-art by up to 193 times.

@article{
 title = {Real-Time LiDAR Point-Cloud Moving Object Segmentation for Autonomous Driving},
 type = {article},
 year = {2023},
 keywords = {CNN,FPGA,LiDAR,moving object segmentation},
 volume = {23},
 month = {1},
 publisher = {MDPI},
 day = {1},
 id = {bfbd9e5f-7d17-3541-be87-8adc1700856f},
 created = {2025-01-20T13:31:38.883Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:36:01.913Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {15 cites},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {The key to autonomous navigation in unmanned systems is the ability to recognize static and moving objects in the environment and to support the task of predicting the future state of the environment, avoiding collisions, and planning. However, because the existing 3D LiDAR point-cloud moving object segmentation (MOS) convolutional neural network (CNN) models are very complex and have large computation burden, it is difficult to perform real-time processing on embedded platforms. In this paper, we propose a lightweight MOS network structure based on LiDAR point-cloud sequence range images with only 2.3 M parameters, which is 66% less than the state-of-the-art network. When running on RTX 3090 GPU, the processing time is 35.82 ms per frame and it achieves an intersection-over-union(IoU) score of 51.3% on the SemanticKITTI dataset. In addition, the proposed CNN successfully runs the FPGA platform using an NVDLA-like hardware architecture, and the system achieves efficient and accurate moving-object segmentation of LiDAR point clouds at a speed of 32 fps, meeting the real-time requirements of autonomous vehicles.},
 bibtype = {article},
 author = {Xie, Xing and Wei, Haowen and Yang, Yongjie},
 doi = {10.3390/s23010547},
 journal = {Sensors},
 number = {1}
}

The key to autonomous navigation in unmanned systems is the ability to recognize static and moving objects in the environment and to support the task of predicting the future state of the environment, avoiding collisions, and planning. However, because the existing 3D LiDAR point-cloud moving object segmentation (MOS) convolutional neural network (CNN) models are very complex and have large computation burden, it is difficult to perform real-time processing on embedded platforms. In this paper, we propose a lightweight MOS network structure based on LiDAR point-cloud sequence range images with only 2.3 M parameters, which is 66% less than the state-of-the-art network. When running on RTX 3090 GPU, the processing time is 35.82 ms per frame and it achieves an intersection-over-union(IoU) score of 51.3% on the SemanticKITTI dataset. In addition, the proposed CNN successfully runs the FPGA platform using an NVDLA-like hardware architecture, and the system achieves efficient and accurate moving-object segmentation of LiDAR point clouds at a speed of 32 fps, meeting the real-time requirements of autonomous vehicles.

@article{
 title = {FlexCNN: An End-to-end Framework for Composing CNN Accelerators on FPGA},
 type = {article},
 year = {2023},
 keywords = {Additional Key Words and PhrasesFPGA,CNN,E-Net,ONNX,OpenPose,U-Net,dilated convolution,systolic array,transposed convolution},
 volume = {16},
 month = {3},
 publisher = {Association for Computing Machinery},
 day = {11},
 id = {117db2ec-47a0-31ce-9802-e73e77e3d86f},
 created = {2025-04-07T08:22:49.355Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:37:15.236Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {44 cites},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {With reduced data reuse and parallelism, recent convolutional neural networks (CNNs) create new challenges for FPGA acceleration. Systolic arrays (SAs) are efficient, scalable architectures for convolutional layers, but without proper optimizations, their efficiency drops dramatically for reasons: (1) the different dimensions within same-type layers, (2) the different convolution layers especially transposed and dilated convolutions, and (3) CNN's complex dataflow graph. Furthermore, significant overheads arise when integrating FPGAs into machine learning frameworks. Therefore, we present a flexible, composable architecture called FlexCNN, which delivers high computation efficiency by employing dynamic tiling, layer fusion, and data layout optimizations. Additionally, we implement a novel versatile SA to process normal, transposed, and dilated convolutions efficiently. FlexCNN also uses a fully pipelined software-hardware integration that alleviates the software overheads. Moreover, with an automated compilation flow, FlexCNN takes a CNN in the ONNX1 representation, performs a design space exploration, and generates an FPGA accelerator. The framework is tested using three complex CNNs: OpenPose, U-Net, and E-Net. The architecture optimizations achieve 2.3× performance improvement. Compared to a standard SA, the versatile SA achieves close-to-ideal speedups, with up to 5.98× and 13.42× for transposed and dilated convolutions, with a 6% average area overhead. The pipelined integration leads to a 5× speedup for OpenPose.},
 bibtype = {article},
 author = {Basalama, Suhail and Sohrabizadeh, Atefeh and Wang, Jie and Guo, Licheng and Cong, Jason},
 doi = {10.1145/3570928},
 journal = {ACM Transactions on Reconfigurable Technology and Systems},
 number = {2}
}

With reduced data reuse and parallelism, recent convolutional neural networks (CNNs) create new challenges for FPGA acceleration. Systolic arrays (SAs) are efficient, scalable architectures for convolutional layers, but without proper optimizations, their efficiency drops dramatically for reasons: (1) the different dimensions within same-type layers, (2) the different convolution layers especially transposed and dilated convolutions, and (3) CNN's complex dataflow graph. Furthermore, significant overheads arise when integrating FPGAs into machine learning frameworks. Therefore, we present a flexible, composable architecture called FlexCNN, which delivers high computation efficiency by employing dynamic tiling, layer fusion, and data layout optimizations. Additionally, we implement a novel versatile SA to process normal, transposed, and dilated convolutions efficiently. FlexCNN also uses a fully pipelined software-hardware integration that alleviates the software overheads. Moreover, with an automated compilation flow, FlexCNN takes a CNN in the ONNX1 representation, performs a design space exploration, and generates an FPGA accelerator. The framework is tested using three complex CNNs: OpenPose, U-Net, and E-Net. The architecture optimizations achieve 2.3× performance improvement. Compared to a standard SA, the versatile SA achieves close-to-ideal speedups, with up to 5.98× and 13.42× for transposed and dilated convolutions, with a 6% average area overhead. The pipelined integration leads to a 5× speedup for OpenPose.

@article{
 title = {Design of an Efficient CNN-Based Cough Detection System on Lightweight FPGA},
 type = {article},
 year = {2023},
 keywords = {CNN,Cough Detection,FPGA,deep learning,hardware acceleration},
 pages = {116-128},
 volume = {17},
 month = {2},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {ae257383-e6a8-33e0-a157-b194f152e913},
 created = {2025-04-07T08:22:50.067Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:22:10.241Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {15 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Precisely and automatically detecting the cough sound is of vital clinical importance. Nevertheless, due to privacy protection considerations, transmitting the raw audio data to the cloud is not permitted, and therefore there is a great demand for an efficient, accurate, and low-cost solution at the edge device. To address this challenge, we propose a semi-custom software-hardware co-design methodology to help build the cough detection system. Specifically, we first design a scalable and compact convolutional neural network (CNN) structure that generates many network instances. Second, we develop a dedicated hardware accelerator to perform the inference computation efficiently, and then we find the optimal network instance by applying network design space exploration. Finally, we compile the optimal network and let it run on the hardware accelerator. The experimental results demonstrate that our model achieves 88.8% classification accuracy, 91.2% sensitivity, 86.5% specificity, and 86.5% precision, while the computation complexity is only 1.09 M multiply-accumulation (MAC). Additionally, when implemented on a lightweight field programmable gate array (FPGA), the complete cough detection system only occupies 7.9 K lookup tables (LUTs), 12.9 K flip-flops (FFs), and 41 digital signal processing (DSP) slices, providing 8.3 GOP/s actual inference throughput and total power dissipation of 0.93 W. This framework meets the needs of partial application and can be easily extended or integrated into other healthcare applications.},
 bibtype = {article},
 author = {Peng, Peng and Jiang, Kai and You, Mingyu and Xie, Jialin and Zhou, Hongjun and Xu, Weisheng and Lu, Jicheng and Li, Xiayu and Xu, Yun},
 doi = {10.1109/TBCAS.2023.3236976},
 journal = {IEEE Transactions on Biomedical Circuits and Systems},
 number = {1}
}

Precisely and automatically detecting the cough sound is of vital clinical importance. Nevertheless, due to privacy protection considerations, transmitting the raw audio data to the cloud is not permitted, and therefore there is a great demand for an efficient, accurate, and low-cost solution at the edge device. To address this challenge, we propose a semi-custom software-hardware co-design methodology to help build the cough detection system. Specifically, we first design a scalable and compact convolutional neural network (CNN) structure that generates many network instances. Second, we develop a dedicated hardware accelerator to perform the inference computation efficiently, and then we find the optimal network instance by applying network design space exploration. Finally, we compile the optimal network and let it run on the hardware accelerator. The experimental results demonstrate that our model achieves 88.8% classification accuracy, 91.2% sensitivity, 86.5% specificity, and 86.5% precision, while the computation complexity is only 1.09 M multiply-accumulation (MAC). Additionally, when implemented on a lightweight field programmable gate array (FPGA), the complete cough detection system only occupies 7.9 K lookup tables (LUTs), 12.9 K flip-flops (FFs), and 41 digital signal processing (DSP) slices, providing 8.3 GOP/s actual inference throughput and total power dissipation of 0.93 W. This framework meets the needs of partial application and can be easily extended or integrated into other healthcare applications.

@phdthesis{
 title = {A Performance Comparison of Path Tracing on FPGA and GPU},
 type = {phdthesis},
 year = {2023},
 websites = {https://urn.kb.se/resolve?urn=urn%3Anbn%3Ase%3Akth%3Adiva-333669},
 city = {Stockholm, Sweden},
 institution = {KTH, School of Electrical Engineering and Computer Science (EECS)},
 id = {16abb70f-3382-39c6-92ff-d1fb07d5b406},
 created = {2025-04-07T08:23:15.908Z},
 accessed = {2025-04-07},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:39:50.893Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {0 cites},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Ray Tracing algorithms have long been the popular choice for rendering realistic images, and in recent years they have also reached the field of real time computer graphics. Although their performance has seen great improvement, they are still very computationally costly to perform, both in terms of time and power. Improving their performance and efficiency is therefore a very relevant field of research, to improve current applications and open up new possibilities for the future. In this degree project we have focused on implementing a ray tracing algorithm called Path Tracing on an Field Programmable Gate Array (FPGA), a type of reconfigurable hardware. FPGAs provide a new path towards increasing performance as we approach the end of Moore’s Law and reach the limits of general-purpose computing, making them an interesting candidate for ray tracing. This study resulted in a substantial performance difference between FPGA and GPU, with GPU outperforming FPGA by around 20 times in full path tracing.},
 bibtype = {phdthesis},
 author = {Lilja, Anton and Videfors, Markus}
}

Ray Tracing algorithms have long been the popular choice for rendering realistic images, and in recent years they have also reached the field of real time computer graphics. Although their performance has seen great improvement, they are still very computationally costly to perform, both in terms of time and power. Improving their performance and efficiency is therefore a very relevant field of research, to improve current applications and open up new possibilities for the future. In this degree project we have focused on implementing a ray tracing algorithm called Path Tracing on an Field Programmable Gate Array (FPGA), a type of reconfigurable hardware. FPGAs provide a new path towards increasing performance as we approach the end of Moore’s Law and reach the limits of general-purpose computing, making them an interesting candidate for ray tracing. This study resulted in a substantial performance difference between FPGA and GPU, with GPU outperforming FPGA by around 20 times in full path tracing.

@inproceedings{
 title = {An Efficient Accelerator for Deep Learning-based Point Cloud Registration on FPGAs},
 type = {inproceedings},
 year = {2023},
 keywords = {FPGA,Point Cloud Registration,PointNet},
 pages = {68-75},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {e6546517-cc4d-3e65-88d2-9255a2e65618},
 created = {2025-04-07T10:09:18.258Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:33:17.687Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {5 cites<br/><br/>Point cloud registration using custom PointNet IP <br/><br/>Xilinx <b>Vitis HLS</b> and <b>Vivado </b>Design Suite on Avnet Ultra96v2 and Xilinx ZCU104},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Point cloud registration is the basis for many robotic applications such as odometry and Simultaneous Localization And Mapping (SLAM), which are increasingly important for autonomous mobile robots. The limitation of computational resources and power budgets on such robots motivates us to study the resource-efficient registration method on low-cost edge devices. In this paper, we propose an FPGA-based novel pipeline for 3D point cloud registration built upon a recent deep learning-based method, PointNetLK. Based on the profiling results, we focus on the PointNet feature extraction as it becomes a major bottleneck; we improve its scalability and memory-efficiency by consuming each input point one-by-one in a pipelined manner instead of processing the whole point cloud at once. We then design a fully-parallelized and pipelined accelerator consisting of a custom PointNet IP core, which fits within both low-cost and mid-range FPGAs (e.g., Avnet Ultra96v2 and Xilinx ZCU104). Experimental results show that our proposed pipeline achieves up to 21.34x and 69.60x faster registration speed than the vanilla PointNetLK and ICP, respectively, while only consuming 722mW and maintaining the same level of accuracy.},
 bibtype = {inproceedings},
 author = {Sugiura, Keisuke and Matsutani, Hiroki},
 doi = {10.1109/PDP59025.2023.00018},
 booktitle = {Proceedings - 2023 31st Euromicro International Conference on Parallel, Distributed and Network-Based Processing, PDP 2023}
}

Point cloud registration is the basis for many robotic applications such as odometry and Simultaneous Localization And Mapping (SLAM), which are increasingly important for autonomous mobile robots. The limitation of computational resources and power budgets on such robots motivates us to study the resource-efficient registration method on low-cost edge devices. In this paper, we propose an FPGA-based novel pipeline for 3D point cloud registration built upon a recent deep learning-based method, PointNetLK. Based on the profiling results, we focus on the PointNet feature extraction as it becomes a major bottleneck; we improve its scalability and memory-efficiency by consuming each input point one-by-one in a pipelined manner instead of processing the whole point cloud at once. We then design a fully-parallelized and pipelined accelerator consisting of a custom PointNet IP core, which fits within both low-cost and mid-range FPGAs (e.g., Avnet Ultra96v2 and Xilinx ZCU104). Experimental results show that our proposed pipeline achieves up to 21.34x and 69.60x faster registration speed than the vanilla PointNetLK and ICP, respectively, while only consuming 722mW and maintaining the same level of accuracy.

@inproceedings{
 title = {A Comparative Analysis of HDL and HLS for Developing CNN Accelerators},
 type = {inproceedings},
 year = {2023},
 keywords = {HDL,HLS},
 pages = {1060-1065},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {0c162dda-bd9e-3438-a670-3db2242e6622},
 created = {2025-04-07T14:08:33.139Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:35:50.028Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {9 cites<br/><br/>HDL implementation of CNN, <br/>comparison with<b> HDL </b>generated vs <b>HLS </b>tool},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {In today's world, complexity is increasing and market demands new development tools for Field Programmable Gate array (FPGA) in Convolution Neural Networks (CNN) applications. CNN involves huge number of computations. Hardware accelerators like FPGA, Graphics Processing Unit (GPU)and Application-Specific Integrated Circuit (ASIC) designs are required to speedup CNN computation. Any CNN architecture in FPGA performs better by increasing speed. This paper presents a comparative study between High Level Synthesis (HLS) and Hardware Description Language (HDL) for implementing CNN architecture in FPGA for developing hardware accelerators. In addition to Hardware Description Language (HDL), High Level Synthesis (HLS) tools are there to increase the higher abstraction level. The FPGA design turn around time is greatly reduced using HLS. High level synthesis efficiently builds and verify hardware design with less time. HLS give better control over optimization of design architecture. HLS provides greater design space exploration. HLS tool accelerates the verification time over Register Transfer Level (RTL) by increasing the abstraction level. The designed Convolution layer using Unrolling and Pipelining techniques for CNN hardware accelerator results are compared with HDL implementation using Artix-7FPGA,XC7A35T-1CPG236C.The results indicate that the HDL version is marginally superior in terms of Resource usage to the HLS implementation. HDL approach does require more programming work than its HLS counterpart. By using all available pragmas and Directives it can achieve comparable level of performance with HDL},
 bibtype = {inproceedings},
 author = {Srilakshmi, S. and Madhumati, G. L.},
 doi = {10.1109/ICAIS56108.2023.10073746},
 booktitle = {Proceedings of the 3rd International Conference on Artificial Intelligence and Smart Energy, ICAIS 2023}
}

In today's world, complexity is increasing and market demands new development tools for Field Programmable Gate array (FPGA) in Convolution Neural Networks (CNN) applications. CNN involves huge number of computations. Hardware accelerators like FPGA, Graphics Processing Unit (GPU)and Application-Specific Integrated Circuit (ASIC) designs are required to speedup CNN computation. Any CNN architecture in FPGA performs better by increasing speed. This paper presents a comparative study between High Level Synthesis (HLS) and Hardware Description Language (HDL) for implementing CNN architecture in FPGA for developing hardware accelerators. In addition to Hardware Description Language (HDL), High Level Synthesis (HLS) tools are there to increase the higher abstraction level. The FPGA design turn around time is greatly reduced using HLS. High level synthesis efficiently builds and verify hardware design with less time. HLS give better control over optimization of design architecture. HLS provides greater design space exploration. HLS tool accelerates the verification time over Register Transfer Level (RTL) by increasing the abstraction level. The designed Convolution layer using Unrolling and Pipelining techniques for CNN hardware accelerator results are compared with HDL implementation using Artix-7FPGA,XC7A35T-1CPG236C.The results indicate that the HDL version is marginally superior in terms of Resource usage to the HLS implementation. HDL approach does require more programming work than its HLS counterpart. By using all available pragmas and Directives it can achieve comparable level of performance with HDL

@article{
 title = {An FPGA smart camera implementation of segmentation models for drone wildfire imagery},
 type = {article},
 year = {2023},
 websites = {http://arxiv.org/abs/2309.01318},
 month = {9},
 day = {3},
 id = {a4f6b51e-69fd-328b-a700-0bf862bfbfa5},
 created = {2025-04-11T07:05:11.387Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-04-11T07:05:25.878Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {5 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6},
 private_publication = {false},
 abstract = {Wildfires represent one of the most relevant natural disasters worldwide, due to their impact on various societal and environmental levels. Thus, a significant amount of research has been carried out to investigate and apply computer vision techniques to address this problem. One of the most promising approaches for wildfire fighting is the use of drones equipped with visible and infrared cameras for the detection, monitoring, and fire spread assessment in a remote manner but in close proximity to the affected areas. However, implementing effective computer vision algorithms on board is often prohibitive since deploying full-precision deep learning models running on GPU is not a viable option, due to their high power consumption and the limited payload a drone can handle. Thus, in this work, we posit that smart cameras, based on low-power consumption field-programmable gate arrays (FPGAs), in tandem with binarized neural networks (BNNs), represent a cost-effective alternative for implementing onboard computing on the edge. Herein we present the implementation of a segmentation model applied to the Corsican Fire Database. We optimized an existing U-Net model for such a task and ported the model to an edge device (a Xilinx Ultra96-v2 FPGA). By pruning and quantizing the original model, we reduce the number of parameters by 90%. Furthermore, additional optimizations enabled us to increase the throughput of the original model from 8 frames per second (FPS) to 33.63 FPS without loss in the segmentation performance: our model obtained 0.912 in Matthews correlation coefficient (MCC),0.915 in F1 score and 0.870 in Hafiane quality index (HAF), and comparable qualitative segmentation results when contrasted to the original full-precision model. The final model was integrated into a low-cost FPGA, which was used to implement a neural network accelerator.},
 bibtype = {article},
 author = {Guarduño-Martinez, Eduardo and Ciprian-Sanchez, Jorge and Valente, Gerardo and Vazquez-Garcia, undefined and Rodriguez-Hernandez, Gerardo and Palacios-Rosas, Adriana and Rossi-Tisson, Lucile and Ochoa-Ruiz, Gilberto}
}

Wildfires represent one of the most relevant natural disasters worldwide, due to their impact on various societal and environmental levels. Thus, a significant amount of research has been carried out to investigate and apply computer vision techniques to address this problem. One of the most promising approaches for wildfire fighting is the use of drones equipped with visible and infrared cameras for the detection, monitoring, and fire spread assessment in a remote manner but in close proximity to the affected areas. However, implementing effective computer vision algorithms on board is often prohibitive since deploying full-precision deep learning models running on GPU is not a viable option, due to their high power consumption and the limited payload a drone can handle. Thus, in this work, we posit that smart cameras, based on low-power consumption field-programmable gate arrays (FPGAs), in tandem with binarized neural networks (BNNs), represent a cost-effective alternative for implementing onboard computing on the edge. Herein we present the implementation of a segmentation model applied to the Corsican Fire Database. We optimized an existing U-Net model for such a task and ported the model to an edge device (a Xilinx Ultra96-v2 FPGA). By pruning and quantizing the original model, we reduce the number of parameters by 90%. Furthermore, additional optimizations enabled us to increase the throughput of the original model from 8 frames per second (FPS) to 33.63 FPS without loss in the segmentation performance: our model obtained 0.912 in Matthews correlation coefficient (MCC),0.915 in F1 score and 0.870 in Hafiane quality index (HAF), and comparable qualitative segmentation results when contrasted to the original full-precision model. The final model was integrated into a low-cost FPGA, which was used to implement a neural network accelerator.

@article{
 title = {An Energy-Efficient Stream-Based FPGA Implementation of Feature Extraction Algorithm for LiDAR Point Clouds with Effective Local-Search},
 type = {article},
 year = {2023},
 keywords = {FPGA,Feature extraction,SLAM,local-search,point cloud},
 pages = {253-265},
 volume = {70},
 month = {1},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {b991ea5e-cdc5-312e-aaca-32a949e714aa},
 created = {2025-06-03T16:08:23.815Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:33:56.471Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {9 cites},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Feature extraction is a fundamental and essential step in light detection and ranging (LiDAR) based simultaneously localization and mapping (SLAM) algorithms. Considering the run-time requirement of feature extraction and the stringent battery constraint in smart vehicles, it is a great challenge to develop fast and highly energy-efficient feature extraction implementation for massive point clouds. Unfortunately, existing implementations not only fail to exploit the available parallelism but also fail to make full use of the local information to optimize the computations. To solve the issue, we propose three novel techniques to achieve a fast and energy-efficient FPGA implementation of the feature extraction algorithm with effective local search. First, we propose a low-complexity projection method and a column-scanning scheduler to organize the irregular and sparse point cloud into a well-organized point cloud matrix. Second, based on the point cloud matrix, we exploit its local information and propose a high-parallel method to detect the coarse-grain feature points. Third, we propose a high-parallel conditional priority queue to progressively and evenly select the fine-grain feature points. Experimental results on the KITTI dataset show that our method implemented on the ZCU104 FPGA board achieves the best accuracy and reaches 584 frames per second (FPS) for the feature extraction of a 64-laser LiDAR's point cloud. Moreover, our proposal achieves the best energy efficiency, which is on average 11.7 times and 9.0 times higher than the state-of-the-art implementations on the GPU and FPGA platforms, respectively.},
 bibtype = {article},
 author = {Sun, Hao and Deng, Qi and Liu, Xinzhe and Shu, Yuhao and Ha, Yajun},
 doi = {10.1109/TCSI.2022.3212075},
 journal = {IEEE Transactions on Circuits and Systems I: Regular Papers},
 number = {1}
}

Feature extraction is a fundamental and essential step in light detection and ranging (LiDAR) based simultaneously localization and mapping (SLAM) algorithms. Considering the run-time requirement of feature extraction and the stringent battery constraint in smart vehicles, it is a great challenge to develop fast and highly energy-efficient feature extraction implementation for massive point clouds. Unfortunately, existing implementations not only fail to exploit the available parallelism but also fail to make full use of the local information to optimize the computations. To solve the issue, we propose three novel techniques to achieve a fast and energy-efficient FPGA implementation of the feature extraction algorithm with effective local search. First, we propose a low-complexity projection method and a column-scanning scheduler to organize the irregular and sparse point cloud into a well-organized point cloud matrix. Second, based on the point cloud matrix, we exploit its local information and propose a high-parallel method to detect the coarse-grain feature points. Third, we propose a high-parallel conditional priority queue to progressively and evenly select the fine-grain feature points. Experimental results on the KITTI dataset show that our method implemented on the ZCU104 FPGA board achieves the best accuracy and reaches 584 frames per second (FPS) for the feature extraction of a 64-laser LiDAR's point cloud. Moreover, our proposal achieves the best energy efficiency, which is on average 11.7 times and 9.0 times higher than the state-of-the-art implementations on the GPU and FPGA platforms, respectively.

@article{
 title = {Graph-based deep learning for communication networks: A survey},
 type = {article},
 year = {2022},
 keywords = {Communication network,Deep learning,Graph,Graph Neural Network,Software Defined Networking},
 pages = {40-54},
 volume = {185},
 id = {00ef2aa4-8ad1-380b-9827-56c04267fc2c},
 created = {2022-03-01T12:39:37.991Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-01T12:39:41.915Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {Communication networks are important infrastructures in contemporary society. There are still many challenges that are not fully solved and new solutions are proposed continuously in this active research area. In recent years, to model the network topology, graph-based deep learning has achieved the state-of-the-art performance in a series of problems in communication networks. In this survey, we review the rapidly growing body of research using different graph-based deep learning models, e.g. graph convolutional and graph attention networks, in various problems from different types of communication networks, e.g. wireless networks, wired networks, and software defined networks. We also present a well-organized list of the problem and solution for each study and identify future research directions. To the best of our knowledge, this paper is the first survey that focuses on the application of graph-based deep learning methods in communication networks involving both wired and wireless scenarios. To track the follow-up research, a public GitHub repository is created, where the relevant papers will be updated continuously.},
 bibtype = {article},
 author = {Jiang, Weiwei},
 doi = {10.1016/j.comcom.2021.12.015},
 journal = {Computer Communications}
}

Communication networks are important infrastructures in contemporary society. There are still many challenges that are not fully solved and new solutions are proposed continuously in this active research area. In recent years, to model the network topology, graph-based deep learning has achieved the state-of-the-art performance in a series of problems in communication networks. In this survey, we review the rapidly growing body of research using different graph-based deep learning models, e.g. graph convolutional and graph attention networks, in various problems from different types of communication networks, e.g. wireless networks, wired networks, and software defined networks. We also present a well-organized list of the problem and solution for each study and identify future research directions. To the best of our knowledge, this paper is the first survey that focuses on the application of graph-based deep learning methods in communication networks involving both wired and wireless scenarios. To track the follow-up research, a public GitHub repository is created, where the relevant papers will be updated continuously.

@article{
 title = {Overhead Reduction for Graph-Based Point Cloud Delivery Using Non-Uniform Quantization},
 type = {article},
 year = {2022},
 id = {13089d7b-32dc-3ac3-b429-8838b67f77fd},
 created = {2022-03-02T07:02:50.194Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-02T07:03:04.305Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 bibtype = {article},
 author = {Electric, Mitsubishi}
}

@article{
 title = {44444 BottleFit : Learning Compressed Representations in Deep Neural Networks for Effective and Efficient Split Computing},
 type = {article},
 year = {2022},
 id = {6777c76c-e9a9-3352-8502-8a24e7d4133f},
 created = {2022-03-04T09:00:49.998Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T07:18:42.939Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449,a6e140dd-a959-4148-9ed8-b29d0c7966c6},
 private_publication = {false},
 bibtype = {article},
 author = {Callegaro, Davide and Levorato, Marco}
}

@article{
 title = {Unsupervised Learning on 3D Point Clouds by Clustering and Contrasting},
 type = {article},
 year = {2022},
 pages = {1-11},
 volume = {14},
 websites = {http://arxiv.org/abs/2202.02543},
 id = {2a89ccc6-008a-3a8d-94ec-567f6ce83b06},
 created = {2022-03-10T14:04:37.039Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-18T10:02:58.606Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5,5041aa39-a3cf-45bd-ada3-df1401e124f1},
 private_publication = {false},
 abstract = {Learning from unlabeled or partially labeled data to alleviate human labeling remains a challenging research topic in 3D modeling. Along this line, unsupervised representation learning is a promising direction to auto-extract features without human intervention. This paper proposes a general unsupervised approach, named \textbfConClu, to perform the learning of point-wise and global features by jointly leveraging point-level clustering and instance-level contrasting. Specifically, for one thing, we design an Expectation-Maximization (EM) like soft clustering algorithm that provides local supervision to extract discriminating local features based on optimal transport. We show that this criterion extends standard cross-entropy minimization to an optimal transport problem, which we solve efficiently using a fast variant of the Sinkhorn-Knopp algorithm. For another, we provide an instance-level contrasting method to learn the global geometry, which is formulated by maximizing the similarity between two augmentations of one point cloud. Experimental evaluations on downstream applications such as 3D object classification and semantic segmentation demonstrate the effectiveness of our framework and show that it can outperform state-of-the-art techniques.},
 bibtype = {article},
 author = {Mei, Guofeng and Yu, Litao and Wu, Qiang and Zhang, Jian and Bennamoun, Mohammed},
 number = {8}
}

Learning from unlabeled or partially labeled data to alleviate human labeling remains a challenging research topic in 3D modeling. Along this line, unsupervised representation learning is a promising direction to auto-extract features without human intervention. This paper proposes a general unsupervised approach, named \textbfConClu, to perform the learning of point-wise and global features by jointly leveraging point-level clustering and instance-level contrasting. Specifically, for one thing, we design an Expectation-Maximization (EM) like soft clustering algorithm that provides local supervision to extract discriminating local features based on optimal transport. We show that this criterion extends standard cross-entropy minimization to an optimal transport problem, which we solve efficiently using a fast variant of the Sinkhorn-Knopp algorithm. For another, we provide an instance-level contrasting method to learn the global geometry, which is formulated by maximizing the similarity between two augmentations of one point cloud. Experimental evaluations on downstream applications such as 3D object classification and semantic segmentation demonstrate the effectiveness of our framework and show that it can outperform state-of-the-art techniques.

@article{
 title = {Anytime 3D Object Reconstruction Using},
 type = {article},
 year = {2022},
 pages = {2162-2169},
 volume = {7},
 publisher = {IEEE},
 id = {e6e45b22-ff4f-3616-baaf-a915c24f5123},
 created = {2022-03-23T06:17:59.519Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-30T07:21:33.125Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Autoencoder2022},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 bibtype = {article},
 author = {Yu, undefined},
 number = {2}
}

@article{
 title = {Geometric Transformer for Fast and Robust Point Cloud Registration},
 type = {article},
 year = {2022},
 websites = {https://arxiv.org/abs/2202.06688v2},
 month = {2},
 day = {14},
 id = {d3d1576f-8b97-3f6c-ab4f-614194058b2e},
 created = {2022-03-28T07:19:05.800Z},
 accessed = {2022-03-28},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-31T06:33:43.733Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {a3e73416-fb6e-4382-ad74-08c4dd4c0e94},
 private_publication = {false},
 abstract = {We study the problem of extracting accurate correspondences for point cloud
registration. Recent keypoint-free methods bypass the detection of repeatable
keypoints which is difficult in low-overlap scenarios, showing great potential
in registration. They seek correspondences over downsampled superpoints, which
are then propagated to dense points. Superpoints are matched based on whether
their neighboring patches overlap. Such sparse and loose matching requires
contextual features capturing the geometric structure of the point clouds. We
propose Geometric Transformer to learn geometric feature for robust superpoint
matching. It encodes pair-wise distances and triplet-wise angles, making it
robust in low-overlap cases and invariant to rigid transformation. The
simplistic design attains surprisingly high matching accuracy such that no
RANSAC is required in the estimation of alignment transformation, leading to
$100$ times acceleration. Our method improves the inlier ratio by $17\sim30$
percentage points and the registration recall by over $7$ points on the
challenging 3DLoMatch benchmark. Our code and models are available at
\urlhttps://github.com/qinzheng93/GeoTransformer.},
 bibtype = {article},
 author = {Qin, Zheng and Yu, Hao and Wang, Changjian and Guo, Yulan and Peng, Yuxing and Xu, Kai},
 doi = {10.48550/arxiv.2202.06688}
}

We study the problem of extracting accurate correspondences for point cloud registration. Recent keypoint-free methods bypass the detection of repeatable keypoints which is difficult in low-overlap scenarios, showing great potential in registration. They seek correspondences over downsampled superpoints, which are then propagated to dense points. Superpoints are matched based on whether their neighboring patches overlap. Such sparse and loose matching requires contextual features capturing the geometric structure of the point clouds. We propose Geometric Transformer to learn geometric feature for robust superpoint matching. It encodes pair-wise distances and triplet-wise angles, making it robust in low-overlap cases and invariant to rigid transformation. The simplistic design attains surprisingly high matching accuracy such that no RANSAC is required in the estimation of alignment transformation, leading to $100$ times acceleration. Our method improves the inlier ratio by $17\sim30$ percentage points and the registration recall by over $7$ points on the challenging 3DLoMatch benchmark. Our code and models are available at \urlhttps://github.com/qinzheng93/GeoTransformer.

@article{
 title = {Geometric Transformer for Fast and Robust Point Cloud Registration},
 type = {article},
 year = {2022},
 websites = {https://arxiv.org/abs/2202.06688v2},
 month = {2},
 day = {14},
 id = {a48947ef-c596-36d9-86ae-e4e46f8afa3c},
 created = {2022-03-28T07:20:03.106Z},
 accessed = {2022-03-28},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T06:51:19.995Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 private_publication = {false},
 abstract = {We study the problem of extracting accurate correspondences for point cloud
registration. Recent keypoint-free methods bypass the detection of repeatable
keypoints which is difficult in low-overlap scenarios, showing great potential
in registration. They seek correspondences over downsampled superpoints, which
are then propagated to dense points. Superpoints are matched based on whether
their neighboring patches overlap. Such sparse and loose matching requires
contextual features capturing the geometric structure of the point clouds. We
propose Geometric Transformer to learn geometric feature for robust superpoint
matching. It encodes pair-wise distances and triplet-wise angles, making it
robust in low-overlap cases and invariant to rigid transformation. The
simplistic design attains surprisingly high matching accuracy such that no
RANSAC is required in the estimation of alignment transformation, leading to
$100$ times acceleration. Our method improves the inlier ratio by $17\sim30$
percentage points and the registration recall by over $7$ points on the
challenging 3DLoMatch benchmark. Our code and models are available at
\urlhttps://github.com/qinzheng93/GeoTransformer.},
 bibtype = {article},
 author = {Qin, Zheng and Yu, Hao and Wang, Changjian and Guo, Yulan and Peng, Yuxing and Xu, Kai},
 doi = {10.48550/arxiv.2202.06688}
}

We study the problem of extracting accurate correspondences for point cloud registration. Recent keypoint-free methods bypass the detection of repeatable keypoints which is difficult in low-overlap scenarios, showing great potential in registration. They seek correspondences over downsampled superpoints, which are then propagated to dense points. Superpoints are matched based on whether their neighboring patches overlap. Such sparse and loose matching requires contextual features capturing the geometric structure of the point clouds. We propose Geometric Transformer to learn geometric feature for robust superpoint matching. It encodes pair-wise distances and triplet-wise angles, making it robust in low-overlap cases and invariant to rigid transformation. The simplistic design attains surprisingly high matching accuracy such that no RANSAC is required in the estimation of alignment transformation, leading to $100$ times acceleration. Our method improves the inlier ratio by $17\sim30$ percentage points and the registration recall by over $7$ points on the challenging 3DLoMatch benchmark. Our code and models are available at \urlhttps://github.com/qinzheng93/GeoTransformer.

@article{
 title = {Anytime 3D Object Reconstruction Using Multi-Modal Variational Autoencoder},
 type = {article},
 year = {2022},
 keywords = {3D object reconstruction,Decoding,Estimation,Real-time systems,Shape,Three-dimensional displays,Training,Visualization,anytime algorithm,data imputation,multi-modal variational autoencoder},
 pages = {2162-2169},
 volume = {7},
 month = {4},
 id = {c0654fc0-7760-378e-98bf-49a4a3500953},
 created = {2022-03-28T09:45:04.459Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:07:46.713Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {yuAnytime3DObject2022},
 source_type = {article},
 notes = {Conference Name: IEEE Robotics and Automation Letters},
 private_publication = {false},
 abstract = {For effective human-robot teaming, it is important for the robots to be able to share their visual perception with the human operators. In a harsh remote collaboration setting, data compression techniques such as autoencoder can be utilized to obtain and transmit the data in terms of latent variables in a compact form. In addition, to ensure real-time runtime performance even under unstable environments, an anytime estimation approach is desired that can reconstruct the full contents from incomplete information. In this context, we propose a method for imputation of latent variables whose elements are partially lost. To achieve the anytime property with only a few dimensions of variables, exploiting prior information of the category-level is essential. A prior distribution used in variational autoencoders is simply assumed to be isotropic Gaussian regardless of the labels of each training datapoint. This type of flattened prior makes it difficult to perform imputation from the category-level distributions. We overcome this limitation by exploiting a category-specific multi-modal prior distribution in the latent space. The missing elements of the partially transferred data can be sampled, by finding a specific modal according to the remaining elements. Since the method is designed to use partial elements for anytime estimation, it can also be applied for data over-compression. Based on the experiments on the ModelNet and Pascal3D datasets, the proposed approach shows consistently superior performance over autoencoder and variational autoencoder up to 70\% data loss. The software is open source and is available from our repository1.},
 bibtype = {article},
 author = {Yu, Hyeonwoo and Oh, Jean},
 doi = {10.1109/LRA.2022.3142439},
 journal = {IEEE Robotics and Automation Letters},
 number = {2}
}

For effective human-robot teaming, it is important for the robots to be able to share their visual perception with the human operators. In a harsh remote collaboration setting, data compression techniques such as autoencoder can be utilized to obtain and transmit the data in terms of latent variables in a compact form. In addition, to ensure real-time runtime performance even under unstable environments, an anytime estimation approach is desired that can reconstruct the full contents from incomplete information. In this context, we propose a method for imputation of latent variables whose elements are partially lost. To achieve the anytime property with only a few dimensions of variables, exploiting prior information of the category-level is essential. A prior distribution used in variational autoencoders is simply assumed to be isotropic Gaussian regardless of the labels of each training datapoint. This type of flattened prior makes it difficult to perform imputation from the category-level distributions. We overcome this limitation by exploiting a category-specific multi-modal prior distribution in the latent space. The missing elements of the partially transferred data can be sampled, by finding a specific modal according to the remaining elements. Since the method is designed to use partial elements for anytime estimation, it can also be applied for data over-compression. Based on the experiments on the ModelNet and Pascal3D datasets, the proposed approach shows consistently superior performance over autoencoder and variational autoencoder up to 70\% data loss. The software is open source and is available from our repository1.

@article{
 title = {Hierarchical Graph-Convolutional Variational AutoEncoding for Generative Modelling of Human Motion},
 type = {article},
 year = {2022},
 keywords = {Computer Science - Artificial Intelligence,Computer Science - Computer Vision and Pattern Re,Computer Science - Machine Learning,Mathematics - Probability},
 websites = {http://arxiv.org/abs/2111.12602},
 month = {1},
 id = {fdf7fca7-30dc-3a22-8fd2-54432c3635a5},
 created = {2022-03-28T09:45:06.188Z},
 accessed = {2022-03-22},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-01T09:16:25.004Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {bourachedHierarchicalGraphConvolutionalVariational2022},
 source_type = {article},
 notes = {arXiv: 2111.12602},
 private_publication = {false},
 abstract = {Models of human motion commonly focus either on trajectory prediction or action classification but rarely both. The marked heterogeneity and intricate compositionality of human motion render each task vulnerable to the data degradation and distributional shift common to real-world scenarios. A sufficiently expressive generative model of action could in theory enable data conditioning and distributional resilience within a unified framework applicable to both tasks. Here we propose a novel architecture based on hierarchical variational autoencoders and deep graph convolutional neural networks for generating a holistic model of action over multiple time-scales. We show this Hierarchical Graph-convolutional Variational Autoencoder (HG-VAE) to be capable of generating coherent actions, detecting out-of-distribution data, and imputing missing data by gradient ascent on the model's posterior. Trained and evaluated on H3.6M and the largest collection of open source human motion data, AMASS, we show HG-VAE can facilitate downstream discriminative learning better than baseline models.},
 bibtype = {article},
 author = {Bourached, Anthony and Gray, Robert and Griffiths, Ryan-Rhys and Jha, Ashwani and Nachev, Parashkev},
 journal = {arXiv:2111.12602 [cs, math]}
}

Models of human motion commonly focus either on trajectory prediction or action classification but rarely both. The marked heterogeneity and intricate compositionality of human motion render each task vulnerable to the data degradation and distributional shift common to real-world scenarios. A sufficiently expressive generative model of action could in theory enable data conditioning and distributional resilience within a unified framework applicable to both tasks. Here we propose a novel architecture based on hierarchical variational autoencoders and deep graph convolutional neural networks for generating a holistic model of action over multiple time-scales. We show this Hierarchical Graph-convolutional Variational Autoencoder (HG-VAE) to be capable of generating coherent actions, detecting out-of-distribution data, and imputing missing data by gradient ascent on the model's posterior. Trained and evaluated on H3.6M and the largest collection of open source human motion data, AMASS, we show HG-VAE can facilitate downstream discriminative learning better than baseline models.

@article{
 title = {A proposal of edge detection in images with multiplicative noise using the Ant Colony System algorithm},
 type = {article},
 year = {2022},
 keywords = {Ant Colony System,Coefficient of variation,Edge detection,Multiplicative noise},
 pages = {104715},
 volume = {110},
 websites = {https://doi.org/10.1016/j.engappai.2022.104715},
 publisher = {Elsevier Ltd},
 id = {6d2fa72f-c12c-3608-b7ba-d85c1d5b9fe9},
 created = {2022-04-05T05:35:07.814Z},
 file_attached = {false},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:07.814Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Multiplicative noise is one of the most aggressive types of noise present in various types of images: Synthetic Aperture Radar, Ultrasound images, Ultrasonic Imaging, among others. Edges detectors such as Canny or Sobel are not very efficient for processing an image with multiplicative noise, they also require filtering algorithms, a preprocessing, which is why bio-inspired algorithms are an alternative for processing images with the presence of multiplicative noise, due to its efficiency in finding an approximate solution. This article proposes a method for the edges detection in images with multiplicative noise using the Ant Colony System algorithm. For which we must adapt the Ant Colony System algorithm to detect contours, this we define the calculation of a global pheromone matrix between several edge detection equations, gradient, and the coefficient of variation, these equations are compared for their edge detection performance using a visual inspection and a performance function. The results of the experiments show a correct implementation of the algorithm proposed to the images with multiplicative noise even for high noise levels.},
 bibtype = {article},
 author = {Baltierra, Sergio and Valdebenito, Jonathan and Mora, Marco},
 doi = {10.1016/j.engappai.2022.104715},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {February}
}

Multiplicative noise is one of the most aggressive types of noise present in various types of images: Synthetic Aperture Radar, Ultrasound images, Ultrasonic Imaging, among others. Edges detectors such as Canny or Sobel are not very efficient for processing an image with multiplicative noise, they also require filtering algorithms, a preprocessing, which is why bio-inspired algorithms are an alternative for processing images with the presence of multiplicative noise, due to its efficiency in finding an approximate solution. This article proposes a method for the edges detection in images with multiplicative noise using the Ant Colony System algorithm. For which we must adapt the Ant Colony System algorithm to detect contours, this we define the calculation of a global pheromone matrix between several edge detection equations, gradient, and the coefficient of variation, these equations are compared for their edge detection performance using a visual inspection and a performance function. The results of the experiments show a correct implementation of the algorithm proposed to the images with multiplicative noise even for high noise levels.

@article{
 title = {Deep Architectures for Image Compression: A Critical Review},
 type = {article},
 year = {2022},
 keywords = {CNN,DNN,Deep learning,Image compression,Review,Survey},
 pages = {108346},
 volume = {191},
 websites = {https://doi.org/10.1016/j.sigpro.2021.108346},
 publisher = {Elsevier B.V.},
 id = {69c71055-7c3d-3584-9266-0b4d85a31944},
 created = {2022-04-05T05:35:07.934Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-07T06:10:54.609Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {Deep learning architectures are now pervasive and filled almost all applications under image processing, computer vision, and biometrics. The attractive property of feature extraction of CNN has solved a lot of conventional image processing problems with much-improved performance & efficiency. The paper aimed to review over a hundred recent state-of-the-art techniques exploiting mostly lossy image compression using deep learning architectures. These deep learning algorithms consists of various architectures like CNN, RNN, GAN, autoencoders and variational autoencoders. We have classified all the algorithms under certain categories for the better and deep understanding. The review is written keeping in mind the contributions of researchers & the challenges faced by them. Various findings for the researchers along with some future directions for a new researcher have been significantly highlighted. Most of the papers reviewed in the compression domain are from the last four years using different methodologies. The review has been summarized by dropping a new outlook for researchers in the realm of image compression.},
 bibtype = {article},
 author = {Mishra, Dipti and Singh, Satish Kumar and Singh, Rajat Kumar},
 doi = {10.1016/j.sigpro.2021.108346},
 journal = {Signal Processing}
}

Deep learning architectures are now pervasive and filled almost all applications under image processing, computer vision, and biometrics. The attractive property of feature extraction of CNN has solved a lot of conventional image processing problems with much-improved performance & efficiency. The paper aimed to review over a hundred recent state-of-the-art techniques exploiting mostly lossy image compression using deep learning architectures. These deep learning algorithms consists of various architectures like CNN, RNN, GAN, autoencoders and variational autoencoders. We have classified all the algorithms under certain categories for the better and deep understanding. The review is written keeping in mind the contributions of researchers & the challenges faced by them. Various findings for the researchers along with some future directions for a new researcher have been significantly highlighted. Most of the papers reviewed in the compression domain are from the last four years using different methodologies. The review has been summarized by dropping a new outlook for researchers in the realm of image compression.

@article{
 title = {A comprehensive survey of clustering algorithms: State-of-the-art machine learning applications, taxonomy, challenges, and future research prospects},
 type = {article},
 year = {2022},
 keywords = {Automatic clustering,Clustering,Clustering algorithms, partitioning,Data mining,Hierarchical clustering,K-Means,Optimization algorithms, Machine learning,Supervised learning,Unsupervised learning},
 pages = {104743},
 volume = {110},
 websites = {https://doi.org/10.1016/j.engappai.2022.104743},
 publisher = {Elsevier Ltd},
 id = {cebed498-d956-3357-8a0e-6cdd88e9fa4f},
 created = {2022-04-05T05:35:07.964Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-07T06:10:54.809Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {Clustering is an essential tool in data mining research and applications. It is the subject of active research in many fields of study, such as computer science, data science, statistics, pattern recognition, artificial intelligence, and machine learning. Several clustering techniques have been proposed and implemented, and most of them successfully find excellent quality or optimal clustering results in the domains mentioned earlier. However, there has been a gradual shift in the choice of clustering methods among domain experts and practitioners alike, which is precipitated by the fact that most traditional clustering algorithms still depend on the number of clusters provided a priori. These conventional clustering algorithms cannot effectively handle real-world data clustering analysis problems where the number of clusters in data objects cannot be easily identified. Also, they cannot effectively manage problems where the optimal number of clusters for a high-dimensional dataset cannot be easily determined. Therefore, there is a need for improved, flexible, and efficient clustering techniques. Recently, a variety of efficient clustering algorithms have been proposed in the literature, and these algorithms produced good results when evaluated on real-world clustering problems. This study presents an up-to-date systematic and comprehensive review of traditional and state-of-the-art clustering techniques for different domains. This survey considers clustering from a more practical perspective. It shows the outstanding role of clustering in various disciplines, such as education, marketing, medicine, biology, and bioinformatics. It also discusses the application of clustering to different fields attracting intensive efforts among the scientific community, such as big data, artificial intelligence, and robotics. This survey paper will be beneficial for both practitioners and researchers. It will serve as a good reference point for researchers and practitioners to design improved and efficient state-of-the-art clustering algorithms.},
 bibtype = {article},
 author = {Ezugwu, Absalom E. and Ikotun, Abiodun M. and Oyelade, Olaide O. and Abualigah, Laith and Agushaka, Jeffery O. and Eke, Christopher I. and Akinyelu, Andronicus A.},
 doi = {10.1016/j.engappai.2022.104743},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {February}
}

Clustering is an essential tool in data mining research and applications. It is the subject of active research in many fields of study, such as computer science, data science, statistics, pattern recognition, artificial intelligence, and machine learning. Several clustering techniques have been proposed and implemented, and most of them successfully find excellent quality or optimal clustering results in the domains mentioned earlier. However, there has been a gradual shift in the choice of clustering methods among domain experts and practitioners alike, which is precipitated by the fact that most traditional clustering algorithms still depend on the number of clusters provided a priori. These conventional clustering algorithms cannot effectively handle real-world data clustering analysis problems where the number of clusters in data objects cannot be easily identified. Also, they cannot effectively manage problems where the optimal number of clusters for a high-dimensional dataset cannot be easily determined. Therefore, there is a need for improved, flexible, and efficient clustering techniques. Recently, a variety of efficient clustering algorithms have been proposed in the literature, and these algorithms produced good results when evaluated on real-world clustering problems. This study presents an up-to-date systematic and comprehensive review of traditional and state-of-the-art clustering techniques for different domains. This survey considers clustering from a more practical perspective. It shows the outstanding role of clustering in various disciplines, such as education, marketing, medicine, biology, and bioinformatics. It also discusses the application of clustering to different fields attracting intensive efforts among the scientific community, such as big data, artificial intelligence, and robotics. This survey paper will be beneficial for both practitioners and researchers. It will serve as a good reference point for researchers and practitioners to design improved and efficient state-of-the-art clustering algorithms.

@article{
 title = {IntroVAC: Introspective Variational Classifiers for learning interpretable latent subspaces},
 type = {article},
 year = {2022},
 keywords = {Attribute manipulation,Autoencoders,Deep learning,Interpretability},
 pages = {104658},
 volume = {109},
 websites = {https://doi.org/10.1016/j.engappai.2021.104658},
 publisher = {Elsevier Ltd},
 id = {8d07907f-4b5d-362f-bee4-61a1c7419a06},
 created = {2022-04-05T05:35:08.068Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:31.615Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Learning useful representations of complex data has been the subject of extensive research for many years. In particular, with the diffusion of complex Deep Learning-based approaches in engineering applications, the possibility to interpret, to a certain degree, model predictions is of fundamental importance for both the model users and developers. In the context of Deep Neural Networks, Variational Autoencoders have gained lots of attention since they provide an explicit model of the data distribution based on an encoder/decoder architecture which is able to both generate images and encode them in a low-dimensional subspace. However, the latent space is not easily interpretable and the generation capabilities show some limitations since images typically look blurry and lack details. In this paper, we propose the Introspective Variational Classifier (IntroVAC), a model that learns interpretable latent subspaces by exploiting information from an additional label and provides improved image quality thanks to an adversarial training strategy. We show that IntroVAC is able to learn meaningful directions in the latent space enabling fine-grained manipulation of image attributes. We validated our approach on the CelebA dataset. When compared with standard Variational Autoencoder Classifiers, the proposed approach outperform them by achieving a Frechét Inception Distance of 25.5 versus a value of 63.9.},
 bibtype = {article},
 author = {Maggipinto, Marco and Terzi, Matteo and Susto, Gian Antonio},
 doi = {10.1016/j.engappai.2021.104658},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {April 2021}
}

Learning useful representations of complex data has been the subject of extensive research for many years. In particular, with the diffusion of complex Deep Learning-based approaches in engineering applications, the possibility to interpret, to a certain degree, model predictions is of fundamental importance for both the model users and developers. In the context of Deep Neural Networks, Variational Autoencoders have gained lots of attention since they provide an explicit model of the data distribution based on an encoder/decoder architecture which is able to both generate images and encode them in a low-dimensional subspace. However, the latent space is not easily interpretable and the generation capabilities show some limitations since images typically look blurry and lack details. In this paper, we propose the Introspective Variational Classifier (IntroVAC), a model that learns interpretable latent subspaces by exploiting information from an additional label and provides improved image quality thanks to an adversarial training strategy. We show that IntroVAC is able to learn meaningful directions in the latent space enabling fine-grained manipulation of image attributes. We validated our approach on the CelebA dataset. When compared with standard Variational Autoencoder Classifiers, the proposed approach outperform them by achieving a Frechét Inception Distance of 25.5 versus a value of 63.9.

@article{
 title = {A comprehensive survey on 3D face recognition methods},
 type = {article},
 year = {2022},
 keywords = {3D face recognition,Deep learning,Expression,Occlusion,Pose,Survey},
 pages = {104669},
 volume = {110},
 websites = {https://doi.org/10.1016/j.engappai.2022.104669},
 publisher = {Elsevier Ltd},
 id = {f6985205-3628-37d9-bcf7-bb3329bd323e},
 created = {2022-04-05T05:35:08.081Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:37.818Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {3D face recognition (3DFR) has emerged as an effective means of characterizing facial identity over the past several decades. Depending on the types of techniques used in recognition, these methods are categorized into traditional and modern. The former generally extract distinctive facial features (e.g. global, local, and hybrid features) for matching, whereas the latter rely primarily on deep learning to perform 3DFR in an end-to-end way. Many literature surveys have been carried out reviewing either traditional or modern methods alone, while only a few studies are conducted simultaneously on both of them. This survey presents a state-of-the-art for 3DFR covering both traditional and modern methods, focusing on the techniques used in face processing, feature extraction, and classification. In addition, we review some specific face recognition challenges, including pose, illumination, expression variations, self-occlusion, and spoofing attack. The commonly used 3D face datasets have been summarized as well.},
 bibtype = {article},
 author = {Li, Menghan and Huang, Bin and Tian, Guohui},
 doi = {10.1016/j.engappai.2022.104669},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {October 2021}
}

3D face recognition (3DFR) has emerged as an effective means of characterizing facial identity over the past several decades. Depending on the types of techniques used in recognition, these methods are categorized into traditional and modern. The former generally extract distinctive facial features (e.g. global, local, and hybrid features) for matching, whereas the latter rely primarily on deep learning to perform 3DFR in an end-to-end way. Many literature surveys have been carried out reviewing either traditional or modern methods alone, while only a few studies are conducted simultaneously on both of them. This survey presents a state-of-the-art for 3DFR covering both traditional and modern methods, focusing on the techniques used in face processing, feature extraction, and classification. In addition, we review some specific face recognition challenges, including pose, illumination, expression variations, self-occlusion, and spoofing attack. The commonly used 3D face datasets have been summarized as well.

@article{
 title = {A novel vision-based weakly supervised framework for autonomous yield estimation in agricultural applications},
 type = {article},
 year = {2022},
 keywords = {Automatic yield estimation,Autonomous systems,Visual learning,Weakly-supervised learning},
 pages = {104615},
 volume = {109},
 websites = {https://doi.org/10.1016/j.engappai.2021.104615},
 publisher = {Elsevier Ltd},
 id = {8d884a27-cef4-339e-952a-134a888f727f},
 created = {2022-04-05T05:35:08.119Z},
 file_attached = {false},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:08.119Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Autonomous systems have been established as a ground-breaking technology in agriculture, particularly for resource optimization and labor savings. However, even those solutions that are limited to monitoring activities, such as yield estimation, rely on costly robotic platforms equipped with a series of range devices (e.g., LIDAR and GPS-RTK). Recently, vision-based strategies have gained considerable attention as a less expensive and more efficient alternative, capable to be on par with or even surpass approaches that benefit from range sensors. Nonetheless, they exploit deep learning methodologies, which require burdensome labeling procedures to perform training. To address these shortcomings, we present a novel approach that performs yield estimation requiring only a monocular camera and needs a limited amount of supervision information. It detects, locates and maps fruits and tree canopies to estimate the total yield of a specific crop. To keep the image labeling effort to a minimum, we propose a weakly-supervision paradigm that only requires a simple binary label encoding the presence or the absence of fruits in the training images. Our approach does not make any assumptions on the underlying platform, i.e., it can be used by collecting images either with a hand-held camera or with an autonomous robot. Therefore, we are able to considerably reduce the deployment time, the energy and the cost of the overall yield estimation system. At the same time, we keep the performance comparable to both vision-based fully supervised baselines (which require costly labeling operations) and classical systems that rely on more expensive and power-demanding sensors.},
 bibtype = {article},
 author = {Bellocchio, Enrico and Crocetti, Francesco and Costante, Gabriele and Fravolini, Mario Luca and Valigi, Paolo},
 doi = {10.1016/j.engappai.2021.104615},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {April 2021}
}

Autonomous systems have been established as a ground-breaking technology in agriculture, particularly for resource optimization and labor savings. However, even those solutions that are limited to monitoring activities, such as yield estimation, rely on costly robotic platforms equipped with a series of range devices (e.g., LIDAR and GPS-RTK). Recently, vision-based strategies have gained considerable attention as a less expensive and more efficient alternative, capable to be on par with or even surpass approaches that benefit from range sensors. Nonetheless, they exploit deep learning methodologies, which require burdensome labeling procedures to perform training. To address these shortcomings, we present a novel approach that performs yield estimation requiring only a monocular camera and needs a limited amount of supervision information. It detects, locates and maps fruits and tree canopies to estimate the total yield of a specific crop. To keep the image labeling effort to a minimum, we propose a weakly-supervision paradigm that only requires a simple binary label encoding the presence or the absence of fruits in the training images. Our approach does not make any assumptions on the underlying platform, i.e., it can be used by collecting images either with a hand-held camera or with an autonomous robot. Therefore, we are able to considerably reduce the deployment time, the energy and the cost of the overall yield estimation system. At the same time, we keep the performance comparable to both vision-based fully supervised baselines (which require costly labeling operations) and classical systems that rely on more expensive and power-demanding sensors.

@article{
 title = {DeltaConv : Anisotropic Geometric Deep Learning with Exterior Calculus},
 type = {article},
 year = {2022},
 pages = {1-12},
 volume = {1},
 id = {bec52d02-416e-3ad6-9aa5-11ce0f2a5f22},
 created = {2022-05-02T08:14:58.363Z},
 file_attached = {false},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-05-02T08:14:58.363Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Wiersma, Ruben},
 number = {1}
}

@article{
 title = {OverlapNet: a siamese network for computing LiDAR scan similarity with applications to loop closing and localization},
 type = {article},
 year = {2022},
 keywords = {Localization,Loop closing,SLAM},
 pages = {61-81},
 volume = {46},
 websites = {https://doi.org/10.1007/s10514-021-09999-0},
 publisher = {Springer US},
 id = {6e49a62f-206c-329e-8b27-f8cac3de6f67},
 created = {2022-07-05T12:32:33.765Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-18T15:28:25.353Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {ed605795-f522-465c-a0b8-5f8a05f7fd5f},
 private_publication = {false},
 abstract = {Localization and mapping are key capabilities of autonomous systems. In this paper, we propose a modified Siamese network to estimate the similarity between pairs of LiDAR scans recorded by autonomous cars. This can be used to address both, loop closing for SLAM and global localization. Our approach utilizes a deep neural network exploiting different cues generated from LiDAR data. It estimates the similarity between pairs of scans using the concept of image overlap generalized to range images and furthermore provides a relative yaw angle estimate. Based on such predictions, our method is able to detect loop closures in a SLAM system or to globally localize in a given map. For loop closure detection, we use the overlap prediction as the similarity measurement to find loop closure candidates and integrate the candidate selection into an existing SLAM system to improve the mapping performance. For global localization, we propose a novel observation model using the predictions provided by OverlapNet and integrate it into a Monte-Carlo localization framework. We evaluate our approach on multiple datasets collected using different LiDAR scanners in various environments. The experimental results show that our method can effectively detect loop closures surpassing the detection performance of state-of-the-art methods and that it generalizes well to different environments. Furthermore, our method reliably localizes a vehicle in typical urban environments globally using LiDAR data collected in different seasons.},
 bibtype = {article},
 author = {Chen, Xieyuanli and Läbe, Thomas and Milioto, Andres and Röhling, Timo and Behley, Jens and Stachniss, Cyrill},
 doi = {10.1007/s10514-021-09999-0},
 journal = {Autonomous Robots},
 number = {1}
}

Localization and mapping are key capabilities of autonomous systems. In this paper, we propose a modified Siamese network to estimate the similarity between pairs of LiDAR scans recorded by autonomous cars. This can be used to address both, loop closing for SLAM and global localization. Our approach utilizes a deep neural network exploiting different cues generated from LiDAR data. It estimates the similarity between pairs of scans using the concept of image overlap generalized to range images and furthermore provides a relative yaw angle estimate. Based on such predictions, our method is able to detect loop closures in a SLAM system or to globally localize in a given map. For loop closure detection, we use the overlap prediction as the similarity measurement to find loop closure candidates and integrate the candidate selection into an existing SLAM system to improve the mapping performance. For global localization, we propose a novel observation model using the predictions provided by OverlapNet and integrate it into a Monte-Carlo localization framework. We evaluate our approach on multiple datasets collected using different LiDAR scanners in various environments. The experimental results show that our method can effectively detect loop closures surpassing the detection performance of state-of-the-art methods and that it generalizes well to different environments. Furthermore, our method reliably localizes a vehicle in typical urban environments globally using LiDAR data collected in different seasons.

@book{
 title = {Attention, please! A survey of neural attention models in deep learning},
 type = {book},
 year = {2022},
 source = {Artificial Intelligence Review},
 keywords = {Attention mechanism,Attention models,Deep learning,Neural networks,Survey},
 issue = {0123456789},
 websites = {https://doi.org/10.1007/s10462-022-10148-x},
 publisher = {Springer Netherlands},
 id = {f50a7c3c-e1d3-3cca-8cae-7eb7ab1008e6},
 created = {2022-07-22T12:20:02.925Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-22T12:20:14.866Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4},
 private_publication = {false},
 abstract = {In humans, Attention is a core property of all perceptual and cognitive operations. Given our limited ability to process competing sources, attention mechanisms select, modulate, and focus on the information most relevant to behavior. For decades, concepts and functions of attention have been studied in philosophy, psychology, neuroscience, and computing. For the last 6 years, this property has been widely explored in deep neural networks. Currently, the state-of-the-art in Deep Learning is represented by neural attention models in several application domains. This survey provides a comprehensive overview and analysis of developments in neural attention models. We systematically reviewed hundreds of architectures in the area, identifying and discussing those in which attention has shown a significant impact. We also developed and made public an automated methodology to facilitate the development of reviews in the area. By critically analyzing 650 works, we describe the primary uses of attention in convolutional, recurrent networks, and generative models, identifying common subgroups of uses and applications. Furthermore, we describe the impact of attention in different application domains and their impact on neural networks’ interpretability. Finally, we list possible trends and opportunities for further research, hoping that this review will provide a succinct overview of the main attentional models in the area and guide researchers in developing future approaches that will drive further improvements.},
 bibtype = {book},
 author = {de Santana Correia, Alana and Colombini, Esther Luna},
 doi = {10.1007/s10462-022-10148-x}
}

In humans, Attention is a core property of all perceptual and cognitive operations. Given our limited ability to process competing sources, attention mechanisms select, modulate, and focus on the information most relevant to behavior. For decades, concepts and functions of attention have been studied in philosophy, psychology, neuroscience, and computing. For the last 6 years, this property has been widely explored in deep neural networks. Currently, the state-of-the-art in Deep Learning is represented by neural attention models in several application domains. This survey provides a comprehensive overview and analysis of developments in neural attention models. We systematically reviewed hundreds of architectures in the area, identifying and discussing those in which attention has shown a significant impact. We also developed and made public an automated methodology to facilitate the development of reviews in the area. By critically analyzing 650 works, we describe the primary uses of attention in convolutional, recurrent networks, and generative models, identifying common subgroups of uses and applications. Furthermore, we describe the impact of attention in different application domains and their impact on neural networks’ interpretability. Finally, we list possible trends and opportunities for further research, hoping that this review will provide a succinct overview of the main attentional models in the area and guide researchers in developing future approaches that will drive further improvements.

@book{
 title = {Machine Learning in Drug Discovery: A Review},
 type = {book},
 year = {2022},
 source = {Artificial Intelligence Review},
 keywords = {Artificial intelligence,Digital pathology,Drug discovery,Machine learning,Prognostic biomarkers,Target validation},
 pages = {1947-1999},
 volume = {55},
 issue = {3},
 websites = {https://doi.org/10.1007/s10462-021-10058-4},
 publisher = {Springer Netherlands},
 id = {897e9a5e-ffe1-306e-9b62-d457de4b87ff},
 created = {2022-07-22T12:20:02.931Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-22T12:20:12.962Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4},
 private_publication = {false},
 abstract = {This review provides the feasible literature on drug discovery through ML tools and techniques that are enforced in every phase of drug development to accelerate the research process and deduce the risk and expenditure in clinical trials. Machine learning techniques improve the decision-making in pharmaceutical data across various applications like QSAR analysis, hit discoveries, de novo drug architectures to retrieve accurate outcomes. Target validation, prognostic biomarkers, digital pathology are considered under problem statements in this review. ML challenges must be applicable for the main cause of inadequacy in interpretability outcomes that may restrict the applications in drug discovery. In clinical trials, absolute and methodological data must be generated to tackle many puzzles in validating ML techniques, improving decision-making, promoting awareness in ML approaches, and deducing risk failures in drug discovery.},
 bibtype = {book},
 author = {Dara, Suresh and Dhamercherla, Swetha and Jadav, Surender Singh and Babu, Ch Madhu and Ahsan, Mohamed Jawed},
 doi = {10.1007/s10462-021-10058-4}
}

This review provides the feasible literature on drug discovery through ML tools and techniques that are enforced in every phase of drug development to accelerate the research process and deduce the risk and expenditure in clinical trials. Machine learning techniques improve the decision-making in pharmaceutical data across various applications like QSAR analysis, hit discoveries, de novo drug architectures to retrieve accurate outcomes. Target validation, prognostic biomarkers, digital pathology are considered under problem statements in this review. ML challenges must be applicable for the main cause of inadequacy in interpretability outcomes that may restrict the applications in drug discovery. In clinical trials, absolute and methodological data must be generated to tackle many puzzles in validating ML techniques, improving decision-making, promoting awareness in ML approaches, and deducing risk failures in drug discovery.

@article{
 title = {3D CAD model retrieval based on sketch and unsupervised variational autoencoder},
 type = {article},
 year = {2022},
 keywords = {Deep learning,Model retrieval,Sketch,Structural semantics,Unsupervised learning},
 pages = {101427},
 volume = {51},
 websites = {https://doi.org/10.1016/j.aei.2021.101427},
 publisher = {Elsevier Ltd},
 id = {2a058129-e787-38c5-b3b9-18766e8711ea},
 created = {2022-07-22T12:20:03.047Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-22T12:20:17.324Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4},
 private_publication = {false},
 abstract = {How to quickly, accurately retrieve and effectively reuse 3D CAD models that conform to user's design intention has become an urgent problem in product design. However, there are several problems with the existing retrieval methods, like not being fast, or accurate, or hard to use. Hence it is difficult to meet the actual needs of the industry. In this paper, we propose a 3D CAD model retrieval approach that considers the speed, accuracy and ease of use at the same time, based on sketches and unsupervised learning. Firstly, the loop is used as the fundamental element of sketch/view, and automatic structural semantics capture algorithms are proposed to extract and construct attributed loop relation tree; Secondly, the recursive neural network based deep variational autoencoders is constructed and optimized to transform arbitrary shapes and sizes of loop relation tree into fixed length descriptor; Finally, based on the fixed length vector descriptor, the sketches and views of 3D CAD models are embedded into the same target feature space, and k-nearest neighbors algorithm is adopted to conduct fast CAD model matching on the feature space. In this manner, a prototype 3D CAD model retrieval system is developed. Experiments on the dataset containing about two thousand 3D CAD models validate the feasibility and effectiveness of the proposed approach.},
 bibtype = {article},
 author = {Qin, Feiwei and Qiu, Shi and Gao, Shuming and Bai, Jing},
 doi = {10.1016/j.aei.2021.101427},
 journal = {Advanced Engineering Informatics},
 number = {August 2021}
}

How to quickly, accurately retrieve and effectively reuse 3D CAD models that conform to user's design intention has become an urgent problem in product design. However, there are several problems with the existing retrieval methods, like not being fast, or accurate, or hard to use. Hence it is difficult to meet the actual needs of the industry. In this paper, we propose a 3D CAD model retrieval approach that considers the speed, accuracy and ease of use at the same time, based on sketches and unsupervised learning. Firstly, the loop is used as the fundamental element of sketch/view, and automatic structural semantics capture algorithms are proposed to extract and construct attributed loop relation tree; Secondly, the recursive neural network based deep variational autoencoders is constructed and optimized to transform arbitrary shapes and sizes of loop relation tree into fixed length descriptor; Finally, based on the fixed length vector descriptor, the sketches and views of 3D CAD models are embedded into the same target feature space, and k-nearest neighbors algorithm is adopted to conduct fast CAD model matching on the feature space. In this manner, a prototype 3D CAD model retrieval system is developed. Experiments on the dataset containing about two thousand 3D CAD models validate the feasibility and effectiveness of the proposed approach.

@article{
 title = {Rotation-Invariant Point Cloud Representation for 3-D Model Recognition},
 type = {article},
 year = {2022},
 keywords = {3-D point cloud,Convolutional neural networks,Data models,Group theory,Harmonic analysis,Point cloud compression,Robot kinematics,Solid modeling,Task analysis,rotation invariant,three-dimensional (3-D) model recognition},
 pages = {1-9},
 publisher = {IEEE},
 id = {52a3dcdf-e7e1-3442-b996-cb7aa4528d17},
 created = {2022-07-28T12:39:24.664Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-28T12:39:41.938Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {353ce2e2-5e70-48e5-951f-78dc31fa40d2},
 private_publication = {false},
 abstract = {Three-dimensional (3-D) data have many applications in the field of computer vision and a point cloud is one of the most popular modalities. Therefore, how to establish a good representation for a point cloud is a core issue in computer vision, especially for 3-D object recognition tasks. Existing approaches mainly focus on the invariance of representation under the group of permutations. However, for point cloud data, it should also be rotation invariant. To address such invariance, in this article, we introduce a relation of equivalence under the action of rotation group, through which the representation of point cloud is located in a homogeneous space. That is, two point clouds are regarded as equivalent when they are only different from a rotation. Our network is flexibly incorporated into existing frameworks for point clouds, which guarantees the proposed approach to be rotation invariant. Besides, a sufficient analysis on how to parameterize the group SO(3) into a convolutional network, which captures a relation with all rotations in 3-D Euclidean space R³. We select the optimal rotation as the best representation of point cloud and propose a solution for minimizing the problem on the rotation group SO(3) by using its geometric structure. To validate the rotation invariance, we combine it with two existing deep models and evaluate them on ModelNet40 dataset and its subset ModelNet10. Experimental results indicate that the proposed strategy improves the performance of those existing deep models when the data involve arbitrary rotations.},
 bibtype = {article},
 author = {Wang, Yan and Zhao, Yining and Ying, Shihui and Du, Shaoyi and Gao, Yue},
 doi = {10.1109/TCYB.2022.3157593},
 journal = {IEEE Transactions on Cybernetics}
}

Three-dimensional (3-D) data have many applications in the field of computer vision and a point cloud is one of the most popular modalities. Therefore, how to establish a good representation for a point cloud is a core issue in computer vision, especially for 3-D object recognition tasks. Existing approaches mainly focus on the invariance of representation under the group of permutations. However, for point cloud data, it should also be rotation invariant. To address such invariance, in this article, we introduce a relation of equivalence under the action of rotation group, through which the representation of point cloud is located in a homogeneous space. That is, two point clouds are regarded as equivalent when they are only different from a rotation. Our network is flexibly incorporated into existing frameworks for point clouds, which guarantees the proposed approach to be rotation invariant. Besides, a sufficient analysis on how to parameterize the group SO(3) into a convolutional network, which captures a relation with all rotations in 3-D Euclidean space R³. We select the optimal rotation as the best representation of point cloud and propose a solution for minimizing the problem on the rotation group SO(3) by using its geometric structure. To validate the rotation invariance, we combine it with two existing deep models and evaluate them on ModelNet40 dataset and its subset ModelNet10. Experimental results indicate that the proposed strategy improves the performance of those existing deep models when the data involve arbitrary rotations.

@article{
 title = {Rotation invariant point cloud analysis: Where local geometry meets global topology},
 type = {article},
 year = {2022},
 keywords = {Classification,Deep learning,Point cloud analysis,Rotation invariance,Segmentation},
 pages = {108626},
 volume = {127},
 websites = {https://doi.org/10.1016/j.patcog.2022.108626},
 publisher = {Elsevier Ltd},
 id = {b0417028-8e58-3572-90e5-b6604cf6d59b},
 created = {2022-07-28T12:39:24.696Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-28T12:39:30.952Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {353ce2e2-5e70-48e5-951f-78dc31fa40d2},
 private_publication = {false},
 abstract = {Point cloud analysis is a fundamental task in 3D computer vision. Most previous works have conducted experiments on synthetic datasets with well-aligned data; while real-world point clouds are often not pre-aligned. How to achieve rotation invariance remains an open problem in point cloud analysis. To meet this challenge, we propose an approach toward achieving rotation-invariant (RI) representations by combining local geometry with global topology. In our local-global-representation (LGR)-Net, we have designed a two-branch network where one stream encodes local geometric RI features and the other encodes global topology-preserving RI features. Motivated by the observation that local geometry and global topology have different yet complementary RI responses in varying regions, two-branch RI features are fused by an innovative multi-layer perceptron (MLP) based attention module. To the best of our knowledge, this work is the first principled approach toward adaptively combining global and local information under the context of RI point cloud analysis. Extensive experiments have demonstrated that our LGR-Net achieves the state-of-the-art performance on various rotation-augmented versions of ModelNet40, ShapeNet, ScanObjectNN, and S3DIS.},
 bibtype = {article},
 author = {Zhao, Chen and Yang, Jiaqi and Xiong, Xin and Zhu, Angfan and Cao, Zhiguo and Li, Xin},
 doi = {10.1016/j.patcog.2022.108626},
 journal = {Pattern Recognition}
}

Point cloud analysis is a fundamental task in 3D computer vision. Most previous works have conducted experiments on synthetic datasets with well-aligned data; while real-world point clouds are often not pre-aligned. How to achieve rotation invariance remains an open problem in point cloud analysis. To meet this challenge, we propose an approach toward achieving rotation-invariant (RI) representations by combining local geometry with global topology. In our local-global-representation (LGR)-Net, we have designed a two-branch network where one stream encodes local geometric RI features and the other encodes global topology-preserving RI features. Motivated by the observation that local geometry and global topology have different yet complementary RI responses in varying regions, two-branch RI features are fused by an innovative multi-layer perceptron (MLP) based attention module. To the best of our knowledge, this work is the first principled approach toward adaptively combining global and local information under the context of RI point cloud analysis. Extensive experiments have demonstrated that our LGR-Net achieves the state-of-the-art performance on various rotation-augmented versions of ModelNet40, ShapeNet, ScanObjectNN, and S3DIS.

@article{
 title = {AGNet: An Attention-Based Graph Network for Point Cloud Classification and Segmentation},
 type = {article},
 year = {2022},
 keywords = {3D point clouds,Geometric features,Graph attention mechanism,Neural network,Shape analysis},
 pages = {1-18},
 volume = {14},
 id = {1917ee21-b0d6-3556-ada4-6c4ebb122ad4},
 created = {2022-08-18T10:53:48.603Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:08.564Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {ed28e2a4-1a3c-4903-8492-0d356dde91f1,244f8db2-6bd4-47d9-8abf-425a263fd4d1},
 private_publication = {false},
 abstract = {Classification and segmentation of point clouds have attracted increasing attention in recent years. On the one hand, it is difficult to extract local features with geometric information. On the other hand, how to select more important features correctly also brings challenges to the research. Therefore, the main challenge in classifying and segmenting the point clouds is how to locate the attentional region. To tackle this challenge, we propose a graph-based neural network with an attention pooling strategy (AGNet). In particular, local feature information can be extracted by constructing a topological structure. Compared to existing methods, AGNet can better extract the spatial information with different distances, and the attentional pooling strategy is capable of selecting the most important features of the topological structure. Therefore, our model can aggregate more information to better represent different point cloud features. We conducted extensive experiments on challenging benchmark datasets including ModelNet40 for object classification, as well as ShapeNet Part and S3DIS for segmentation. Both the quantitative and qualitative experiments demonstrated a consistent advantage for the tasks of point set classification and segmentation.},
 bibtype = {article},
 author = {Jing, Weipeng and Zhang, Wenjun and Li, Linhui and Di, Donglin and Chen, Guangsheng and Wang, Jian},
 doi = {10.3390/rs14041036},
 journal = {Remote Sensing},
 number = {4}
}

Classification and segmentation of point clouds have attracted increasing attention in recent years. On the one hand, it is difficult to extract local features with geometric information. On the other hand, how to select more important features correctly also brings challenges to the research. Therefore, the main challenge in classifying and segmenting the point clouds is how to locate the attentional region. To tackle this challenge, we propose a graph-based neural network with an attention pooling strategy (AGNet). In particular, local feature information can be extracted by constructing a topological structure. Compared to existing methods, AGNet can better extract the spatial information with different distances, and the attentional pooling strategy is capable of selecting the most important features of the topological structure. Therefore, our model can aggregate more information to better represent different point cloud features. We conducted extensive experiments on challenging benchmark datasets including ModelNet40 for object classification, as well as ShapeNet Part and S3DIS for segmentation. Both the quantitative and qualitative experiments demonstrated a consistent advantage for the tasks of point set classification and segmentation.

@article{
 title = {Enhancing Local Feature Learning Using Diffusion for 3D Point Cloud Understanding},
 type = {article},
 year = {2022},
 websites = {http://arxiv.org/abs/2207.01174},
 id = {153d090e-04a7-36bb-9386-4d3e5be12c16},
 created = {2022-08-18T10:53:48.905Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:09.149Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Learning point clouds is challenging due to the lack of connectivity information, i.e., edges. Although existing edge-aware methods can improve the performance by modeling edges, how edges contribute to the improvement is unclear. In this study, we propose a method that automatically learns to enhance/suppress edges while keeping the its working mechanism clear. First, we theoretically figure out how edge enhancement/suppression works. Second, we experimentally verify the edge enhancement/suppression behavior. Third, we empirically show that this behavior improves performance. In general, we observe that the proposed method achieves competitive performance in point cloud classification and segmentation tasks.},
 bibtype = {article},
 author = {Xiu, Haoyi and Liu, Xin and Wang, Weimin and Kim, Kyoung-Sook and Shinohara, Takayuki and Chang, Qiong and Matsuoka, Masashi}
}

Learning point clouds is challenging due to the lack of connectivity information, i.e., edges. Although existing edge-aware methods can improve the performance by modeling edges, how edges contribute to the improvement is unclear. In this study, we propose a method that automatically learns to enhance/suppress edges while keeping the its working mechanism clear. First, we theoretically figure out how edge enhancement/suppression works. Second, we experimentally verify the edge enhancement/suppression behavior. Third, we empirically show that this behavior improves performance. In general, we observe that the proposed method achieves competitive performance in point cloud classification and segmentation tasks.

@article{
 title = {Point3D: tracking actions as moving points with 3D CNNs},
 type = {article},
 year = {2022},
 pages = {1-14},
 websites = {http://arxiv.org/abs/2203.10584},
 id = {a85ba577-590d-38ac-95c5-8f0302f58651},
 created = {2022-08-18T10:53:48.940Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:11.420Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Spatio-temporal action recognition has been a challenging task that involves detecting where and when actions occur. Current state-of-the-art action detectors are mostly anchor-based, requiring sensitive anchor designs and huge computations due to calculating large numbers of anchor boxes. Motivated by nascent anchor-free approaches, we propose Point3D, a flexible and computationally efficient network with high precision for spatio-temporal action recognition. Our Point3D consists of a Point Head for action localization and a 3D Head for action classification. Firstly, Point Head is used to track center points and knot key points of humans to localize the bounding box of an action. These location features are then piped into a time-wise attention to learn long-range dependencies across frames. The 3D Head is later deployed for the final action classification. Our Point3D achieves state-of-the-art performance on the JHMDB, UCF101-24, and AVA benchmarks in terms of frame-mAP and video-mAP. Comprehensive ablation studies also demonstrate the effectiveness of each module proposed in our Point3D.},
 bibtype = {article},
 author = {Mo, Shentong and Xia, Jingfei and Tan, Xiaoqing and Raj, Bhiksha}
}

Spatio-temporal action recognition has been a challenging task that involves detecting where and when actions occur. Current state-of-the-art action detectors are mostly anchor-based, requiring sensitive anchor designs and huge computations due to calculating large numbers of anchor boxes. Motivated by nascent anchor-free approaches, we propose Point3D, a flexible and computationally efficient network with high precision for spatio-temporal action recognition. Our Point3D consists of a Point Head for action localization and a 3D Head for action classification. Firstly, Point Head is used to track center points and knot key points of humans to localize the bounding box of an action. These location features are then piped into a time-wise attention to learn long-range dependencies across frames. The 3D Head is later deployed for the final action classification. Our Point3D achieves state-of-the-art performance on the JHMDB, UCF101-24, and AVA benchmarks in terms of frame-mAP and video-mAP. Comprehensive ablation studies also demonstrate the effectiveness of each module proposed in our Point3D.

@article{
 title = {LPF-Defense: 3D Adversarial Defense based on Frequency Analysis},
 type = {article},
 year = {2022},
 pages = {1-19},
 websites = {http://arxiv.org/abs/2202.11287},
 id = {7c999778-74c5-32eb-a8ca-b59c9be711e2},
 created = {2022-08-29T14:18:24.788Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-04-24T15:41:56.349Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Naderi2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143,f4d8f1ef-fdcb-4a5b-a626-6e2fea47fb6d,ed28e2a4-1a3c-4903-8492-0d356dde91f1,244f8db2-6bd4-47d9-8abf-425a263fd4d1,8c0c38c1-fb54-417d-a224-97fc3b1afba9},
 private_publication = {false},
 abstract = {Although 3D point cloud classification has recently been widely deployed in different application scenarios, it is still very vulnerable to adversarial attacks. This increases the importance of robust training of 3D models in the face of adversarial attacks. Based on our analysis on the performance of existing adversarial attacks, more adversarial perturbations are found in the mid and high-frequency components of input data. Therefore, by suppressing the high-frequency content in the training phase, the models robustness against adversarial examples is improved. Experiments showed that the proposed defense method decreases the success rate of six attacks on PointNet, PointNet++ ,, and DGCNN models. In particular, improvements are achieved with an average increase of classification accuracy by 3.8 % on drop100 attack and 4.26 % on drop200 attack compared to the state-of-the-art methods. The method also improves models accuracy on the original dataset compared to other available methods.},
 bibtype = {article},
 author = {Naderi, Hanieh and Etemadi, Arian and Noorbakhsh, Kimia and Kasaei, Shohreh}
}

Although 3D point cloud classification has recently been widely deployed in different application scenarios, it is still very vulnerable to adversarial attacks. This increases the importance of robust training of 3D models in the face of adversarial attacks. Based on our analysis on the performance of existing adversarial attacks, more adversarial perturbations are found in the mid and high-frequency components of input data. Therefore, by suppressing the high-frequency content in the training phase, the models robustness against adversarial examples is improved. Experiments showed that the proposed defense method decreases the success rate of six attacks on PointNet, PointNet++ ,, and DGCNN models. In particular, improvements are achieved with an average increase of classification accuracy by 3.8 % on drop100 attack and 4.26 % on drop200 attack compared to the state-of-the-art methods. The method also improves models accuracy on the original dataset compared to other available methods.

@article{
 title = {NormalAttack : Curvature-Aware Shape Deformation along Normals for Imperceptible Point Cloud Attack},
 type = {article},
 year = {2022},
 volume = {2022},
 id = {b5e88401-a7d8-3857-ab33-03c70da738ee},
 created = {2022-08-29T14:18:24.794Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-12-05T14:13:52.483Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {db36ed60-3b58-424a-b9a4-a9c7322975f3,8c0c38c1-fb54-417d-a224-97fc3b1afba9},
 private_publication = {false},
 bibtype = {article},
 author = {Tang, Keke and Shi, Yawen and Wu, Jianpeng and Peng, Weilong and Khan, Asad and Zhu, Peican and Gu, Zhaoquan},
 journal = {Security and Communication Networks}
}

@article{
 title = {Point Cloud Attacks in Graph Spectral Domain: When 3D Geometry Meets Graph Signal Processing},
 type = {article},
 year = {2022},
 pages = {1-15},
 volume = {14},
 websites = {http://arxiv.org/abs/2207.13326},
 id = {c85aeb37-a718-321d-a3e8-3f172ae14d5e},
 created = {2022-08-29T14:18:24.873Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-10-03T13:31:11.008Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8f8f5505-1a28-42dd-a82c-92b5738465f1,ed28e2a4-1a3c-4903-8492-0d356dde91f1,244f8db2-6bd4-47d9-8abf-425a263fd4d1,8c0c38c1-fb54-417d-a224-97fc3b1afba9},
 private_publication = {false},
 abstract = {With the increasing attention in various 3D safety-critical applications, point cloud learning models have been shown to be vulnerable to adversarial attacks. Although existing 3D attack methods achieve high success rates, they delve into the data space with point-wise perturbation, which may neglect the geometric characteristics. Instead, we propose point cloud attacks from a new perspective -- the graph spectral domain attack, aiming to perturb graph transform coefficients in the spectral domain that corresponds to varying certain geometric structure. Specifically, leveraging on graph signal processing, we first adaptively transform the coordinates of points onto the spectral domain via graph Fourier transform (GFT) for compact representation. Then, we analyze the influence of different spectral bands on the geometric structure, based on which we propose to perturb the GFT coefficients via a learnable graph spectral filter. Considering the low-frequency components mainly contribute to the rough shape of the 3D object, we further introduce a low-frequency constraint to limit perturbations within imperceptible high-frequency components. Finally, the adversarial point cloud is generated by transforming the perturbed spectral representation back to the data domain via the inverse GFT. Experimental results demonstrate the effectiveness of the proposed attack in terms of both the imperceptibility and attack success rates.},
 bibtype = {article},
 author = {Liu, Daizong and Hu, Wei and Li, Xin},
 number = {8}
}

With the increasing attention in various 3D safety-critical applications, point cloud learning models have been shown to be vulnerable to adversarial attacks. Although existing 3D attack methods achieve high success rates, they delve into the data space with point-wise perturbation, which may neglect the geometric characteristics. Instead, we propose point cloud attacks from a new perspective -- the graph spectral domain attack, aiming to perturb graph transform coefficients in the spectral domain that corresponds to varying certain geometric structure. Specifically, leveraging on graph signal processing, we first adaptively transform the coordinates of points onto the spectral domain via graph Fourier transform (GFT) for compact representation. Then, we analyze the influence of different spectral bands on the geometric structure, based on which we propose to perturb the GFT coefficients via a learnable graph spectral filter. Considering the low-frequency components mainly contribute to the rough shape of the 3D object, we further introduce a low-frequency constraint to limit perturbations within imperceptible high-frequency components. Finally, the adversarial point cloud is generated by transforming the perturbed spectral representation back to the data domain via the inverse GFT. Experimental results demonstrate the effectiveness of the proposed attack in terms of both the imperceptibility and attack success rates.

@article{
 title = {Geometry-Aware Generation of Adversarial Point Clouds},
 type = {article},
 year = {2022},
 pages = {2984-2999},
 volume = {44},
 publisher = {IEEE},
 id = {e716209c-61e4-3a3d-9639-e873ebe3e1c1},
 created = {2022-09-01T14:14:15.402Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-01T14:14:28.435Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Wen, Yuxin and Lin, Jiehong and Chen, Ke and Chen, C L Philip and Jia, Kui},
 number = {6}
}

@article{
 title = {ART-Point : Improving Rotation Robustness of Point Cloud Classifiers via Adversarial Rotation},
 type = {article},
 year = {2022},
 pages = {14371-14380},
 id = {b6832215-ca93-34aa-a2db-e7a42d4a253e},
 created = {2022-09-01T14:14:15.404Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:04.100Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {db36ed60-3b58-424a-b9a4-a9c7322975f3},
 private_publication = {false},
 bibtype = {article},
 author = {Wang, Ruibin and Yang, Yibo and Tao, Dacheng}
}

@article{
 title = {SCONE : Surface Coverage Optimization in Unknown Environments by Volumetric Integration},
 type = {article},
 year = {2022},
 pages = {1-24},
 id = {18e7f89a-67ab-3d2c-9e94-93b86fc79929},
 created = {2022-09-06T11:37:21.049Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-04-24T15:41:56.546Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Guedon2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143,f4d8f1ef-fdcb-4a5b-a626-6e2fea47fb6d,2a64087a-cd32-494a-8140-2abf0b1356c6,40587c46-f934-43d3-a07b-43b15ce182ed},
 private_publication = {false},
 bibtype = {article},
 author = {Guédon, Antoine and Ponts, Ecole}
}

@article{
 title = {Boosting 3D Adversarial Attacks With Attacking on Frequency},
 type = {article},
 year = {2022},
 pages = {50974-50984},
 volume = {10},
 publisher = {IEEE},
 id = {bac6be16-2ba7-3d69-8bc2-1dd7b7da7131},
 created = {2022-09-06T14:02:41.606Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:07.708Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {db36ed60-3b58-424a-b9a4-a9c7322975f3,8c0c38c1-fb54-417d-a224-97fc3b1afba9},
 private_publication = {false},
 bibtype = {article},
 author = {Liu, Binbin and Zhang, Jinlai and Zhu, Jihong},
 doi = {10.1109/ACCESS.2022.3171659},
 journal = {IEEE Access}
}

@article{
 title = {Comprehensive Review of Deep Learning-Based 3D Point Cloud Completion Processing and Analysis},
 type = {article},
 year = {2022},
 pages = {1-22},
 id = {a06ac853-087d-3f56-80f6-099da9844268},
 created = {2022-09-13T13:57:40.428Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-26T08:55:23.175Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {f4318b9a-a9ed-4f3f-8c48-09937099013b},
 private_publication = {false},
 abstract = {Point cloud completion is a generation and estimation issue derived from the partial point clouds, which plays a vital role in the applications in 3D computer vision. The progress of deep learning (DL) has impressively improved the capability and robustness of point cloud completion. However, the quality of completed point clouds is still needed to be further enhanced to meet the practical utilization. Therefore, this work aims to conduct a comprehensive survey on various methods, including point-based, convolution-based, graph-based, and generative model-based approaches, etc. And this survey summarizes the comparisons among these methods to provoke further research insights. Besides, this review sums up the commonly used datasets and illustrates the applications of point cloud completion. Eventually, we also discussed possible research trends in this promptly expanding field.},
 bibtype = {article},
 author = {Fei, Ben and Yang, Weidong and Chen, Wen-Ming and Li, Zhijun and Li, Yikang and Ma, Tao and Hu, Xing and Ma, Lipeng},
 doi = {10.1109/tits.2022.3195555},
 journal = {IEEE Transactions on Intelligent Transportation Systems}
}

Point cloud completion is a generation and estimation issue derived from the partial point clouds, which plays a vital role in the applications in 3D computer vision. The progress of deep learning (DL) has impressively improved the capability and robustness of point cloud completion. However, the quality of completed point clouds is still needed to be further enhanced to meet the practical utilization. Therefore, this work aims to conduct a comprehensive survey on various methods, including point-based, convolution-based, graph-based, and generative model-based approaches, etc. And this survey summarizes the comparisons among these methods to provoke further research insights. Besides, this review sums up the commonly used datasets and illustrates the applications of point cloud completion. Eventually, we also discussed possible research trends in this promptly expanding field.

@article{
 title = {PUFA-GAN: A Frequency-Aware Generative Adversarial Network for 3D Point Cloud Upsampling},
 type = {article},
 year = {2022},
 pages = {1-13},
 websites = {http://arxiv.org/abs/2203.00914},
 id = {63dd1f06-4b7e-3d34-807d-d4fcdbd3b90a},
 created = {2022-09-13T13:57:40.429Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-19T07:34:03.912Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {f4318b9a-a9ed-4f3f-8c48-09937099013b,b6d75013-efe2-4ddc-b3db-65496bd4db9f},
 private_publication = {false},
 abstract = {We propose a generative adversarial network for point cloud upsampling, which can not only make the upsampled points evenly distributed on the underlying surface but also efficiently generate clean high frequency regions. The generator of our network includes a dynamic graph hierarchical residual aggregation unit and a hierarchical residual aggregation unit for point feature extraction and upsampling, respectively. The former extracts multiscale point-wise descriptive features, while the latter captures rich feature details with hierarchical residuals. To generate neat edges, our discriminator uses a graph filter to extract and retain high frequency points. The generated high resolution point cloud and corresponding high frequency points help the discriminator learn the global and high frequency properties of the point cloud. We also propose an identity distribution loss function to make sure that the upsampled points remain on the underlying surface of the input low resolution point cloud. To assess the regularity of the upsampled points in high frequency regions, we introduce two evaluation metrics. Objective and subjective results demonstrate that the visual quality of the upsampled point clouds generated by our method is better than that of the state-of-the-art methods.},
 bibtype = {article},
 author = {Liu, Hao and Yuan, Hui and Hou, Junhui and Hamzaoui, Raouf and Gao, Wei}
}

We propose a generative adversarial network for point cloud upsampling, which can not only make the upsampled points evenly distributed on the underlying surface but also efficiently generate clean high frequency regions. The generator of our network includes a dynamic graph hierarchical residual aggregation unit and a hierarchical residual aggregation unit for point feature extraction and upsampling, respectively. The former extracts multiscale point-wise descriptive features, while the latter captures rich feature details with hierarchical residuals. To generate neat edges, our discriminator uses a graph filter to extract and retain high frequency points. The generated high resolution point cloud and corresponding high frequency points help the discriminator learn the global and high frequency properties of the point cloud. We also propose an identity distribution loss function to make sure that the upsampled points remain on the underlying surface of the input low resolution point cloud. To assess the regularity of the upsampled points in high frequency regions, we introduce two evaluation metrics. Objective and subjective results demonstrate that the visual quality of the upsampled point clouds generated by our method is better than that of the state-of-the-art methods.

@article{
 title = {PU-REFINER : A GEOMETRY REFINER WITH ADVERSARIAL LEARNING FOR POINT CLOUD UPSAMPLING 2 . School of Control Science and Engineering , Shandong University , Jinan , China , 3 . School of Engineering and Sustainable Development , De Montfort University , Lei},
 type = {article},
 year = {2022},
 pages = {2270-2274},
 publisher = {IEEE},
 id = {6b8b41dd-12b5-3f22-8ec5-7cc4295bce61},
 created = {2022-09-19T07:34:03.592Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-19T07:34:15.018Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {f4318b9a-a9ed-4f3f-8c48-09937099013b,b6d75013-efe2-4ddc-b3db-65496bd4db9f},
 private_publication = {false},
 bibtype = {article},
 author = {Liu, Hao and Yuan, Hui and Hamzaoui, Raouf and Gao, Wei and Li, Shuai}
}

@article{
 title = {Geometry-Aware Generation of Adversarial Point Clouds},
 type = {article},
 year = {2022},
 keywords = {Adversarial example,object surface geometry,point cloud},
 pages = {2984-2999},
 volume = {44},
 id = {08e418b9-ab8c-3a58-a370-77839efc1d9f},
 created = {2022-10-10T13:41:15.050Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-10-12T07:53:07.477Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8410cff5-f764-42b2-a9b5-ead8d2dee5c8,8c0c38c1-fb54-417d-a224-97fc3b1afba9},
 private_publication = {false},
 abstract = {Machine learning models have been shown to be vulnerable to adversarial examples. While most of the existing methods for adversarial attack and defense work on the 2D image domain, a few recent attempts have been made to extend them to 3D point cloud data. However, adversarial results obtained by these methods typically contain point outliers, which are both noticeable and easy to defend against using the simple techniques of outlier removal. Motivated by the different mechanisms by which humans perceive 2D images and 3D shapes, in this paper we propose the new design of geometry-aware objectives, whose solutions favor (the discrete versions of) the desired surface properties of smoothness and fairness. To generate adversarial point clouds, we use a targeted attack misclassification loss that supports continuous pursuit of increasingly malicious signals. Regularizing the targeted attack loss with our proposed geometry-aware objectives results in our proposed method, Geometry-Aware Adversarial Attack (GeoA3GeoA3). The results of GeoA3GeoA3 tend to be more harmful, arguably harder to defend against, and of the key adversarial characterization of being imperceptible to humans. While the main focus of this paper is to learn to generate adversarial point clouds, we also present a simple but effective algorithm termed Geo+A3Geo+A3-IterNormPro, with Iterative Normal Projection (IterNorPro) that solves a new objective function Geo+A3Geo+A3, towards surface-level adversarial attacks via generation of adversarial point clouds. We quantitatively evaluate our methods on both synthetic and physical objects in terms of attack success rate and geometric regularity. For a qualitative evaluation, we conduct subjective studies by collecting human preferences from Amazon Mechanical Turk. Comparative results in comprehensive experiments confirm the advantages of our proposed methods. Our source codes are publicly available at https://github.com/Yuxin-Wen/GeoA3.},
 bibtype = {article},
 author = {Wen, Yuxin and Lin, Jiehong and Chen, Ke and Chen, C. L.Philip and Jia, Kui},
 doi = {10.1109/TPAMI.2020.3044712},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
 number = {6}
}

Machine learning models have been shown to be vulnerable to adversarial examples. While most of the existing methods for adversarial attack and defense work on the 2D image domain, a few recent attempts have been made to extend them to 3D point cloud data. However, adversarial results obtained by these methods typically contain point outliers, which are both noticeable and easy to defend against using the simple techniques of outlier removal. Motivated by the different mechanisms by which humans perceive 2D images and 3D shapes, in this paper we propose the new design of geometry-aware objectives, whose solutions favor (the discrete versions of) the desired surface properties of smoothness and fairness. To generate adversarial point clouds, we use a targeted attack misclassification loss that supports continuous pursuit of increasingly malicious signals. Regularizing the targeted attack loss with our proposed geometry-aware objectives results in our proposed method, Geometry-Aware Adversarial Attack (GeoA3GeoA3). The results of GeoA3GeoA3 tend to be more harmful, arguably harder to defend against, and of the key adversarial characterization of being imperceptible to humans. While the main focus of this paper is to learn to generate adversarial point clouds, we also present a simple but effective algorithm termed Geo+A3Geo+A3-IterNormPro, with Iterative Normal Projection (IterNorPro) that solves a new objective function Geo+A3Geo+A3, towards surface-level adversarial attacks via generation of adversarial point clouds. We quantitatively evaluate our methods on both synthetic and physical objects in terms of attack success rate and geometric regularity. For a qualitative evaluation, we conduct subjective studies by collecting human preferences from Amazon Mechanical Turk. Comparative results in comprehensive experiments confirm the advantages of our proposed methods. Our source codes are publicly available at https://github.com/Yuxin-Wen/GeoA3.

@article{
 title = {Robust Object Classification Approach Using Spherical Harmonics},
 type = {article},
 year = {2022},
 keywords = {Object recognition,point cloud classification,robust classification,spherical harmonics},
 pages = {21541-21553},
 volume = {10},
 publisher = {IEEE},
 id = {167d8e3b-4b47-3207-9c15-b458b3231c03},
 created = {2023-04-24T07:38:01.276Z},
 file_attached = {false},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-04-24T15:41:54.977Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Mukhaimar2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143,f4d8f1ef-fdcb-4a5b-a626-6e2fea47fb6d},
 private_publication = {false},
 abstract = {Point clouds produced by either 3D scanners or multi-view images are often imperfect and contain noise or outliers. This paper presents an end-to-end robust spherical harmonics approach to classifying 3D objects. The proposed framework first uses the voxel grid of concentric spheres to learn features over the unit ball. We then limit the spherical harmonics order level to suppress the effect of noise and outliers. In addition, the entire classification operation is performed in the Fourier domain. As a result, our proposed model learned features that are less sensitive to data perturbations and corruptions. We tested our proposed model against several types of data perturbations and corruptions, such as noise and outliers. Our results show that the proposed model has fewer parameters, competes with state-of-art networks in terms of robustness to data inaccuracies, and is faster than other robust methods. Our implementation code is also publicly available at https://github.com/AymanMukh/R-SCNN},
 bibtype = {article},
 author = {Mukhaimar, Ayman and Tennakoon, Ruwan and Lai, Chow Yin and Hoseinnezhad, Reza and Bab-Hadiashar, Alireza},
 doi = {10.1109/ACCESS.2022.3151350},
 journal = {IEEE Access}
}

Point clouds produced by either 3D scanners or multi-view images are often imperfect and contain noise or outliers. This paper presents an end-to-end robust spherical harmonics approach to classifying 3D objects. The proposed framework first uses the voxel grid of concentric spheres to learn features over the unit ball. We then limit the spherical harmonics order level to suppress the effect of noise and outliers. In addition, the entire classification operation is performed in the Fourier domain. As a result, our proposed model learned features that are less sensitive to data perturbations and corruptions. We tested our proposed model against several types of data perturbations and corruptions, such as noise and outliers. Our results show that the proposed model has fewer parameters, competes with state-of-art networks in terms of robustness to data inaccuracies, and is faster than other robust methods. Our implementation code is also publicly available at https://github.com/AymanMukh/R-SCNN

@article{
 title = {Rotation-Invariant Point Cloud Representation for 3-D Model Recognition},
 type = {article},
 year = {2022},
 keywords = {3-D point cloud,Group theory,rotation invariant,three-dimensional (3-D) model recognition},
 pages = {10948-10956},
 volume = {52},
 publisher = {IEEE},
 id = {097a15cb-e368-3d47-9484-63a1f1cd03de},
 created = {2023-05-03T13:16:39.125Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:27.228Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wang2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Three-dimensional (3-D) data have many applications in the field of computer vision and a point cloud is one of the most popular modalities. Therefore, how to establish a good representation for a point cloud is a core issue in computer vision, especially for 3-D object recognition tasks. Existing approaches mainly focus on the invariance of representation under the group of permutations. However, for point cloud data, it should also be rotation invariant. To address such invariance, in this article, we introduce a relation of equivalence under the action of rotation group, through which the representation of point cloud is located in a homogeneous space. That is, two point clouds are regarded as equivalent when they are only different from a rotation. Our network is flexibly incorporated into existing frameworks for point clouds, which guarantees the proposed approach to be rotation invariant. Besides, a sufficient analysis on how to parameterize the group SO(3) into a convolutional network, which captures a relation with all rotations in 3-D Euclidean space R3. We select the optimal rotation as the best representation of point cloud and propose a solution for minimizing the problem on the rotation group SO(3) by using its geometric structure. To validate the rotation invariance, we combine it with two existing deep models and evaluate them on ModelNet40 dataset and its subset ModelNet10. Experimental results indicate that the proposed strategy improves the performance of those existing deep models when the data involve arbitrary rotations.},
 bibtype = {article},
 author = {Wang, Yan and Zhao, Yining and Ying, Shihui and Du, Shaoyi and Gao, Yue},
 doi = {10.1109/TCYB.2022.3157593},
 journal = {IEEE Transactions on Cybernetics},
 number = {10}
}

Three-dimensional (3-D) data have many applications in the field of computer vision and a point cloud is one of the most popular modalities. Therefore, how to establish a good representation for a point cloud is a core issue in computer vision, especially for 3-D object recognition tasks. Existing approaches mainly focus on the invariance of representation under the group of permutations. However, for point cloud data, it should also be rotation invariant. To address such invariance, in this article, we introduce a relation of equivalence under the action of rotation group, through which the representation of point cloud is located in a homogeneous space. That is, two point clouds are regarded as equivalent when they are only different from a rotation. Our network is flexibly incorporated into existing frameworks for point clouds, which guarantees the proposed approach to be rotation invariant. Besides, a sufficient analysis on how to parameterize the group SO(3) into a convolutional network, which captures a relation with all rotations in 3-D Euclidean space R3. We select the optimal rotation as the best representation of point cloud and propose a solution for minimizing the problem on the rotation group SO(3) by using its geometric structure. To validate the rotation invariance, we combine it with two existing deep models and evaluate them on ModelNet40 dataset and its subset ModelNet10. Experimental results indicate that the proposed strategy improves the performance of those existing deep models when the data involve arbitrary rotations.

@article{
 title = {Z2P: Instant Visualization of Point Clouds},
 type = {article},
 year = {2022},
 pages = {461-471},
 volume = {41},
 id = {e5bdbade-8158-3922-aa66-ff7167b0b785},
 created = {2023-05-03T13:16:39.335Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.544Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Metzer2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We present a technique for visualizing point clouds using a neural network. Our technique allows for an instant preview of any point cloud, and bypasses the notoriously difficult surface reconstruction problem or the need to estimate oriented normals for splat-based rendering. We cast the preview problem as a conditional image-to-image translation task, and design a neural network that translates point depth-map directly into an image, where the point cloud is visualized as though a surface was reconstructed from it. Furthermore, the resulting appearance of the visualized point cloud can be, optionally, conditioned on simple control variables (e.g., color and light). We demonstrate that our technique instantly produces plausible images, and can, on-the-fly effectively handle noise, non-uniform sampling, and thin surfaces sheets.},
 bibtype = {article},
 author = {Metzer, G. and Hanocka, R. and Giryes, R. and Mitra, N. J. and Cohen-Or, D.},
 doi = {10.1111/cgf.14487},
 journal = {Computer Graphics Forum},
 number = {2}
}

We present a technique for visualizing point clouds using a neural network. Our technique allows for an instant preview of any point cloud, and bypasses the notoriously difficult surface reconstruction problem or the need to estimate oriented normals for splat-based rendering. We cast the preview problem as a conditional image-to-image translation task, and design a neural network that translates point depth-map directly into an image, where the point cloud is visualized as though a surface was reconstructed from it. Furthermore, the resulting appearance of the visualized point cloud can be, optionally, conditioned on simple control variables (e.g., color and light). We demonstrate that our technique instantly produces plausible images, and can, on-the-fly effectively handle noise, non-uniform sampling, and thin surfaces sheets.

@article{
 title = {Using Spherical Harmonics for Navigating in Dynamic and Uncertain Environments},
 type = {article},
 year = {2022},
 keywords = {Collision Avoidance,Path Planning,Spherical Harmonics},
 pages = {567-572},
 volume = {55},
 websites = {https://doi.org/10.1016/j.ifacol.2022.11.243},
 publisher = {Elsevier Ltd},
 id = {964195eb-adfe-3d05-84bd-5ed94c9e37ef},
 created = {2023-05-03T13:16:39.459Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.720Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Patrick2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {In this paper, we present a novel approach for local motion planning in an unknown environment populated by static and dynamic obstacles. A key component of our planner is the use of harmonic basis functions defined over the unit sphere, known as spherical harmonics (SH), to represent a collision-free space near an agent. In contrast with our previous approach, which only applies to unknown static environments, the approach proposed herein handles dynamic obstacles as well. It is done by efficiently calculating the coefficients of a spherical harmonics approximation of the local collision-free space for each step of a given planning horizon. Our method for approximating the collision-free space with spherical harmonics allows us to plan trajectories in challenging environments (e.g., planning through narrow passages) where other methods for generating search-spaces fail. To accurately approximate the collision-free space for future time steps along a planning horizon, we use a Kalman Filter (KF) to propagate the measured point cloud through time. Our use of the KF allows us to better approximate obstacles moving in space in comparison to inaccurate constant obstacle velocity assumptions used by other methods. Finally, we show the effectiveness of our proposed trajectory planner using two different non-trivial scenarios.},
 bibtype = {article},
 author = {Patrick, Steven D. and Bakolas, Efstathios},
 doi = {10.1016/j.ifacol.2022.11.243},
 journal = {IFAC-PapersOnLine},
 number = {37}
}

In this paper, we present a novel approach for local motion planning in an unknown environment populated by static and dynamic obstacles. A key component of our planner is the use of harmonic basis functions defined over the unit sphere, known as spherical harmonics (SH), to represent a collision-free space near an agent. In contrast with our previous approach, which only applies to unknown static environments, the approach proposed herein handles dynamic obstacles as well. It is done by efficiently calculating the coefficients of a spherical harmonics approximation of the local collision-free space for each step of a given planning horizon. Our method for approximating the collision-free space with spherical harmonics allows us to plan trajectories in challenging environments (e.g., planning through narrow passages) where other methods for generating search-spaces fail. To accurately approximate the collision-free space for future time steps along a planning horizon, we use a Kalman Filter (KF) to propagate the measured point cloud through time. Our use of the KF allows us to better approximate obstacles moving in space in comparison to inaccurate constant obstacle velocity assumptions used by other methods. Finally, we show the effectiveness of our proposed trajectory planner using two different non-trivial scenarios.

@article{
 title = {Equivalence Between SE(3) Equivariant Networks via Steerable Kernels and Group Convolution},
 type = {article},
 year = {2022},
 pages = {1-23},
 websites = {http://arxiv.org/abs/2211.15903},
 id = {498d0053-3454-37ed-9bfa-d0916f795def},
 created = {2023-05-03T13:16:39.505Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.901Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Poulenard2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {A wide range of techniques have been proposed in recent years for designing neural networks for 3D data that are equivariant under rotation and translation of the input. Most approaches for equivariance under the Euclidean group $\mathrmSE(3)$ of rotations and translations fall within one of the two major categories. The first category consists of methods that use $\mathrmSE(3)$-convolution which generalizes classical $\mathbbR^3$-convolution on signals over $\mathrmSE(3)$. Alternatively, it is possible to use \textitsteerable convolution which achieves $\mathrmSE(3)$-equivariance by imposing constraints on $\mathbbR^3$-convolution of tensor fields. It is known by specialists in the field that the two approaches are equivalent, with steerable convolution being the Fourier transform of $\mathrmSE(3)$ convolution. Unfortunately, these results are not widely known and moreover the exact relations between deep learning architectures built upon these two approaches have not been precisely described in the literature on equivariant deep learning. In this work we provide an in-depth analysis of both methods and their equivalence and relate the two constructions to multiview convolutional networks. Furthermore, we provide theoretical justifications of separability of $\mathrmSE(3)$ group convolution, which explain the applicability and success of some recent approaches. Finally, we express different methods using a single coherent formalism and provide explicit formulas that relate the kernels learned by different methods. In this way, our work helps to unify different previously-proposed techniques for achieving roto-translational equivariance, and helps to shed light on both the utility and precise differences between various alternatives. We also derive new TFN non-linearities from our equivalence principle and test them on practical benchmark datasets.},
 bibtype = {article},
 author = {Poulenard, Adrien and Ovsjanikov, Maks and Guibas, Leonidas J.},
 number = {3}
}

A wide range of techniques have been proposed in recent years for designing neural networks for 3D data that are equivariant under rotation and translation of the input. Most approaches for equivariance under the Euclidean group $\mathrmSE(3)$ of rotations and translations fall within one of the two major categories. The first category consists of methods that use $\mathrmSE(3)$-convolution which generalizes classical $\mathbbR^3$-convolution on signals over $\mathrmSE(3)$. Alternatively, it is possible to use \textitsteerable convolution which achieves $\mathrmSE(3)$-equivariance by imposing constraints on $\mathbbR^3$-convolution of tensor fields. It is known by specialists in the field that the two approaches are equivalent, with steerable convolution being the Fourier transform of $\mathrmSE(3)$ convolution. Unfortunately, these results are not widely known and moreover the exact relations between deep learning architectures built upon these two approaches have not been precisely described in the literature on equivariant deep learning. In this work we provide an in-depth analysis of both methods and their equivalence and relate the two constructions to multiview convolutional networks. Furthermore, we provide theoretical justifications of separability of $\mathrmSE(3)$ group convolution, which explain the applicability and success of some recent approaches. Finally, we express different methods using a single coherent formalism and provide explicit formulas that relate the kernels learned by different methods. In this way, our work helps to unify different previously-proposed techniques for achieving roto-translational equivariance, and helps to shed light on both the utility and precise differences between various alternatives. We also derive new TFN non-linearities from our equivalence principle and test them on practical benchmark datasets.

@article{
 title = {Solid waste shape description and generation based on spherical harmonics and probability density function},
 type = {article},
 year = {2022},
 keywords = {accepted 9th august 2021,bezama,by associate editor alberto,dem,probability function,received 18th february 2021,shape description,shape generation,solid waste particle,spherical harmonics},
 id = {338d6a9c-ff7f-34cd-a87f-9b1db9b6507e},
 created = {2023-05-03T13:16:39.646Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.721Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Li2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 bibtype = {article},
 author = {Li, Yifeng and Qin, Xunpeng and Zhang, Zhenyuan and Dong, Huanyu},
 doi = {10.1177/0734242X211045003}
}

@article{
 title = {PSE-Match: A Viewpoint-Free Place Recognition Method With Parallel Semantic Embedding},
 type = {article},
 year = {2022},
 keywords = {3D place recognition,divergence learning,global localization,place feature learning,semantic embedding},
 pages = {11249-11260},
 volume = {23},
 publisher = {IEEE},
 id = {ec37ddb7-e054-30e1-819e-dfa56d29fb92},
 created = {2023-05-03T13:16:39.811Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.192Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Yin2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Accurate localization on the autonomous driving cars is essential for autonomy and driving safety, especially for complex urban streets and search-and-rescue subterranean environments where high-accurate GPS is not available. However current odometry estimation may introduce the drifting problems in long-term navigation without robust global localization. The main challenges involve scene divergence under the interference of dynamic environments and effective perception of observation and object layout variance from different viewpoints. To tackle these challenges, we present PSE-Match, a viewpoint-free place recognition method based on parallel semantic analysis of isolated semantic attributes from 3D point-cloud models. Compared with the original point cloud, the observed variance of semantic attributes is smaller. PSE-Match incorporates a divergence place learning network to capture different semantic attributes parallelly through the spherical harmonics domain. Using both existing benchmark datasets and two in-field collected datasets, our experiments show that the proposed method achieves above 70% average recall with top one retrieval and above 95% average recall with top ten retrieval cases. And PSE-Match has also demonstrated an obvious generalization ability with limited training dataset.},
 bibtype = {article},
 author = {Yin, Peng and Xu, Lingyun and Feng, Ziyue and Egorov, Anton and Li, Bing},
 doi = {10.1109/TITS.2021.3102429},
 journal = {IEEE Transactions on Intelligent Transportation Systems},
 number = {8}
}

Accurate localization on the autonomous driving cars is essential for autonomy and driving safety, especially for complex urban streets and search-and-rescue subterranean environments where high-accurate GPS is not available. However current odometry estimation may introduce the drifting problems in long-term navigation without robust global localization. The main challenges involve scene divergence under the interference of dynamic environments and effective perception of observation and object layout variance from different viewpoints. To tackle these challenges, we present PSE-Match, a viewpoint-free place recognition method based on parallel semantic analysis of isolated semantic attributes from 3D point-cloud models. Compared with the original point cloud, the observed variance of semantic attributes is smaller. PSE-Match incorporates a divergence place learning network to capture different semantic attributes parallelly through the spherical harmonics domain. Using both existing benchmark datasets and two in-field collected datasets, our experiments show that the proposed method achieves above 70% average recall with top one retrieval and above 95% average recall with top ten retrieval cases. And PSE-Match has also demonstrated an obvious generalization ability with limited training dataset.

@article{
 title = {Spherical harmonics to quantify cranial asymmetry in deformational plagiocephaly},
 type = {article},
 year = {2022},
 pages = {1-10},
 volume = {12},
 websites = {https://doi.org/10.1038/s41598-021-04181-z},
 publisher = {Nature Publishing Group UK},
 id = {62fd082a-1da8-3167-b5ce-bebed3dec3b4},
 created = {2023-05-03T13:16:39.843Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.858Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Grieb2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Cranial deformation and deformational plagiocephaly (DP) in particular affect an important percentage of infants. The assessment and diagnosis of the deformation are commonly carried by manual measurements that provide low interuser accuracy. Another approach is the use of three-dimensional (3D) models. Nevertheless, in most cases, deformation measurements are carried out manually on the 3D model. It is necessary to develop methodologies for the detection of DP that are automatic, accurate and take profit on the high quantity of information of the 3D models. Spherical harmonics are proposed as a new methodology to identify DP from head 3D models. The ideal fitted ellipsoid for each head is computed and the orthogonal distances between head and ellipsoid are obtained. Finally, the distances are modelled using spherical harmonics. Spherical harmonic coefficients of degree 2 and order − 2 are identified as the correct ones to represent the asymmetry characteristic of DP. The obtained coefficient is compared to other anthropometric deformation indexes, such as Asymmetry Index, Oblique Cranial Length Ratio, Posterior Asymmetry Index and Anterior Asymmetry Index. The coefficient of degree 2 and order − 2 with a maximum degree of 4 is found to provide better results than the commonly computed anthropometric indexes in the detection of DP.},
 bibtype = {article},
 author = {Grieb, Jonas and Barbero-García, Inés and Lerma, José Luis},
 doi = {10.1038/s41598-021-04181-z},
 journal = {Scientific Reports},
 number = {1}
}

Cranial deformation and deformational plagiocephaly (DP) in particular affect an important percentage of infants. The assessment and diagnosis of the deformation are commonly carried by manual measurements that provide low interuser accuracy. Another approach is the use of three-dimensional (3D) models. Nevertheless, in most cases, deformation measurements are carried out manually on the 3D model. It is necessary to develop methodologies for the detection of DP that are automatic, accurate and take profit on the high quantity of information of the 3D models. Spherical harmonics are proposed as a new methodology to identify DP from head 3D models. The ideal fitted ellipsoid for each head is computed and the orthogonal distances between head and ellipsoid are obtained. Finally, the distances are modelled using spherical harmonics. Spherical harmonic coefficients of degree 2 and order − 2 are identified as the correct ones to represent the asymmetry characteristic of DP. The obtained coefficient is compared to other anthropometric deformation indexes, such as Asymmetry Index, Oblique Cranial Length Ratio, Posterior Asymmetry Index and Anterior Asymmetry Index. The coefficient of degree 2 and order − 2 with a maximum degree of 4 is found to provide better results than the commonly computed anthropometric indexes in the detection of DP.

@article{
 title = {NPBG++: Accelerating Neural Point-Based Graphics},
 type = {article},
 year = {2022},
 keywords = {3D from multi-view and sensors,Deep learning architectures and techniques,Image and video synthesis and generation,Machine learning,Representation learning,Scene analysis and understanding},
 pages = {15948-15958},
 volume = {2022-June},
 id = {63f9f918-2d68-376d-a9f2-119e312b86d9},
 created = {2023-05-03T13:16:39.849Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.102Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Rakhimov2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We present a new system $(NPBG++)$ for the novel view synthesis (NVS) task that achieves high rendering realism with low scene fitting time. Our method efficiently lever-ages the multiview observations and the point cloud of a static scene to predict a neural descriptor for each point, improving upon the pipeline of Neural Point-Based Graph-ics [1] in several important ways. By predicting the descrip-tors with a single pass through the source images, we lift the requirement of per-scene optimization while also making the neural descriptors view-dependent and more suit-able for scenes with strong non-Lambertian effects. In our comparisons, the proposed system outperforms previous NVS approaches in terms of fitting and rendering runtimes while producing images of similar quality. Project page: https://rakhimovv.github.io/npbgpp/.},
 bibtype = {article},
 author = {Rakhimov, Ruslan and Ardelean, Andrei Timotei and Lempitsky, Victor and Burnaev, Evgeny},
 doi = {10.1109/CVPR52688.2022.01550},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

We present a new system $(NPBG++)$ for the novel view synthesis (NVS) task that achieves high rendering realism with low scene fitting time. Our method efficiently lever-ages the multiview observations and the point cloud of a static scene to predict a neural descriptor for each point, improving upon the pipeline of Neural Point-Based Graph-ics [1] in several important ways. By predicting the descrip-tors with a single pass through the source images, we lift the requirement of per-scene optimization while also making the neural descriptors view-dependent and more suit-able for scenes with strong non-Lambertian effects. In our comparisons, the proposed system outperforms previous NVS approaches in terms of fitting and rendering runtimes while producing images of similar quality. Project page: https://rakhimovv.github.io/npbgpp/.

@article{
 title = {Equivariant Point Cloud Analysis via Learning Orientations for Message Passing},
 type = {article},
 year = {2022},
 keywords = {Pose estimation and tracking,Scene analysis and understanding},
 pages = {18910-18919},
 volume = {2022-June},
 id = {3184c9cf-2c09-3926-9ebe-615b1bafdff0},
 created = {2023-05-03T13:16:40.002Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.311Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Luo2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Equivariance has been a long-standing concern in various fields ranging from computer vision to physical modeling. Most previous methods struggle with generality, simplicity, and expressiveness - some are designed ad hoc for specific data types, some are too complex to be accessible, and some sacrifice flexible transformations. In this work, we propose a novel and simple framework to achieve equivariance for point cloud analysis based on the message passing (graph neural network) scheme. We find the equivariant property could be obtained by introducing an orientation for each point to decouple the relative position for each point from the global pose of the entire point cloud. Therefore, we extend current message passing networks with a module that learns orientations for each point. Before aggregating information from the neighbors of a point, the networks transforms the neighbors' coordinates based on the point's learned orientations. We provide formal proofs to show the equivariance of the proposed framework. Empirically, we demonstrate that our proposed method is competitive on both point cloud analysis and physical modeling tasks. Code is available at https://github.com/luost26/Equivariant-OrientedMP.},
 bibtype = {article},
 author = {Luo, Shitong and Li, Jiahan and Guan, Jiaqi and Su, Yufeng and Cheng, Chaoran and Peng, Jian and Ma, Jianzhu},
 doi = {10.1109/CVPR52688.2022.01836},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

Equivariance has been a long-standing concern in various fields ranging from computer vision to physical modeling. Most previous methods struggle with generality, simplicity, and expressiveness - some are designed ad hoc for specific data types, some are too complex to be accessible, and some sacrifice flexible transformations. In this work, we propose a novel and simple framework to achieve equivariance for point cloud analysis based on the message passing (graph neural network) scheme. We find the equivariant property could be obtained by introducing an orientation for each point to decouple the relative position for each point from the global pose of the entire point cloud. Therefore, we extend current message passing networks with a module that learns orientations for each point. Before aggregating information from the neighbors of a point, the networks transforms the neighbors' coordinates based on the point's learned orientations. We provide formal proofs to show the equivariance of the proposed framework. Empirically, we demonstrate that our proposed method is competitive on both point cloud analysis and physical modeling tasks. Code is available at https://github.com/luost26/Equivariant-OrientedMP.

@book{
 title = {Differentiable Point-Based Radiance Fields for Efficient View Synthesis},
 type = {book},
 year = {2022},
 source = {Proceedings - SIGGRAPH Asia 2022 Conference Papers},
 keywords = {Image-based Rendering,Neural Rendering,Novel View Synthesis},
 volume = {41},
 issue = {4},
 publisher = {Association for Computing Machinery},
 id = {d2e9c6df-4f0d-38ee-8689-1f7ebe8e7cd1},
 created = {2023-05-03T13:16:40.146Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.937Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Zhang2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We propose a differentiable rendering algorithm for efficient novel view synthesis. By departing from volume-based representations in favor of a learned point representation, we improve on existing methods more than an order of magnitude in memory and runtime, both in training and inference. The method begins with a uniformly-sampled random point cloud and learns per-point position and view-dependent appearance, using a differentiable splat-based renderer to train the model to reproduce a set of input training images with the given pose. Our method is up to 300 × faster than NeRF in both training and inference, with only a marginal sacrifice in quality, while using less than 10 MB of memory for a static scene. For dynamic scenes, our method trains two orders of magnitude faster than STNeRF and renders at a near interactive rate, while maintaining high image quality and temporal coherence even without imposing any temporal-coherency regularizers.},
 bibtype = {book},
 author = {Zhang, Qiang and Baek, Seung Hwan and Rusinkiewicz, Szymon and Heide, Felix},
 doi = {10.1145/3550469.3555413}
}

We propose a differentiable rendering algorithm for efficient novel view synthesis. By departing from volume-based representations in favor of a learned point representation, we improve on existing methods more than an order of magnitude in memory and runtime, both in training and inference. The method begins with a uniformly-sampled random point cloud and learns per-point position and view-dependent appearance, using a differentiable splat-based renderer to train the model to reproduce a set of input training images with the given pose. Our method is up to 300 × faster than NeRF in both training and inference, with only a marginal sacrifice in quality, while using less than 10 MB of memory for a static scene. For dynamic scenes, our method trains two orders of magnitude faster than STNeRF and renders at a near interactive rate, while maintaining high image quality and temporal coherence even without imposing any temporal-coherency regularizers.

@article{
 title = {Fast Sequence-Matching Enhanced},
 type = {article},
 year = {2022},
 pages = {2127-2135},
 volume = {69},
 publisher = {IEEE},
 id = {6f3ac59d-a5f3-3e31-b370-0fb651da4b00},
 created = {2023-05-03T13:16:40.155Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.887Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Recognition2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 bibtype = {article},
 author = {Recognition, Viewpoint-invariant Place and Yin, Peng and Wang, Fuying and Egorov, Anton and Hou, Jiafan and Jia, Zhenzhong},
 number = {2}
}

@article{
 title = {Surface Eigenvalues with Lattice-Based Approximation In comparison with analytical solution},
 type = {article},
 year = {2022},
 pages = {1-28},
 websites = {http://arxiv.org/abs/2203.03603},
 id = {682e2e10-3322-3077-b4a9-7d6d352cb4bc},
 created = {2023-05-03T13:16:40.694Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.267Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wu2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {In this paper, we propose a meshless method of computing eigenvalues and eigenfunctions of a given surface embedded in $\mathbb R^3$. We use point cloud data as input and generate the lattice approximation for some neighborhood of the surface. We compute the eigenvalues and eigenvectors of the cubic lattice graph as an approximation of the eigenvalues and eigenfunctions of the Laplace-Beltrami operator on the surface. We perform extensive numerical experiments on surfaces with various topology and compare our computed eigenvalues from point cloud surface with exact solutions and standard finite element methods using triangle mesh.},
 bibtype = {article},
 author = {Wu, Yingying and Wu, Tianqi and Yau, Shing-Tung}
}

In this paper, we propose a meshless method of computing eigenvalues and eigenfunctions of a given surface embedded in $\mathbb R^3$. We use point cloud data as input and generate the lattice approximation for some neighborhood of the surface. We compute the eigenvalues and eigenvectors of the cubic lattice graph as an approximation of the eigenvalues and eigenfunctions of the Laplace-Beltrami operator on the surface. We perform extensive numerical experiments on surfaces with various topology and compare our computed eigenvalues from point cloud surface with exact solutions and standard finite element methods using triangle mesh.

@article{
 title = {e3nn: Euclidean Neural Networks},
 type = {article},
 year = {2022},
 pages = {1-22},
 websites = {http://arxiv.org/abs/2207.09453},
 id = {8a3cb5fc-c9d3-31d3-82d6-8f0c4f0325fe},
 created = {2023-05-03T13:16:41.030Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.339Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Geiger2022},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We present e3nn, a generalized framework for creating E(3) equivariant trainable functions, also known as Euclidean neural networks. e3nn naturally operates on geometry and geometric tensors that describe systems in 3D and transform predictably under a change of coordinate system. The core of e3nn are equivariant operations such as the TensorProduct class or the spherical harmonics functions that can be composed to create more complex modules such as convolutions and attention mechanisms. These core operations of e3nn can be used to efficiently articulate Tensor Field Networks, 3D Steerable CNNs, Clebsch-Gordan Networks, SE(3) Transformers and other E(3) equivariant networks.},
 bibtype = {article},
 author = {Geiger, Mario and Smidt, Tess},
 number = {3}
}

We present e3nn, a generalized framework for creating E(3) equivariant trainable functions, also known as Euclidean neural networks. e3nn naturally operates on geometry and geometric tensors that describe systems in 3D and transform predictably under a change of coordinate system. The core of e3nn are equivariant operations such as the TensorProduct class or the spherical harmonics functions that can be composed to create more complex modules such as convolutions and attention mechanisms. These core operations of e3nn can be used to efficiently articulate Tensor Field Networks, 3D Steerable CNNs, Clebsch-Gordan Networks, SE(3) Transformers and other E(3) equivariant networks.

@article{
 title = {SCONE: Surface Coverage Optimization in Unknown Environments by Volumetric Integration},
 type = {article},
 year = {2022},
 websites = {http://arxiv.org/abs/2208.10449},
 id = {3b8f6a29-94eb-3448-9e97-8c5b3504728a},
 created = {2023-06-22T10:06:22.405Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-06-22T10:06:29.191Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {cbcad584-0c50-48fe-a0d7-5b4c781cba83},
 private_publication = {false},
 abstract = {Next Best View computation (NBV) is a long-standing problem in robotics, and consists in identifying the next most informative sensor position(s) for reconstructing a 3D object or scene efficiently and accurately. Like most current methods, we consider NBV prediction from a depth sensor like Lidar systems. Learning-based methods relying on a volumetric representation of the scene are suitable for path planning, but have lower accuracy than methods using a surface-based representation. However, the latter do not scale well with the size of the scene and constrain the camera to a small number of poses. To obtain the advantages of both representations, we show that we can maximize surface metrics by Monte Carlo integration over a volumetric representation. In particular, we propose an approach, SCONE, that relies on two neural modules: The first module predicts occupancy probability in the entire volume of the scene. Given any new camera pose, the second module samples points in the scene based on their occupancy probability and leverages a self-attention mechanism to predict the visibility of the samples. Finally, we integrate the visibility to evaluate the gain in surface coverage for the new camera pose. NBV is selected as the pose that maximizes the gain in total surface coverage. Our method scales to large scenes and handles free camera motion: It takes as input an arbitrarily large point cloud gathered by a depth sensor as well as camera poses to predict NBV. We demonstrate our approach on a novel dataset made of large and complex 3D scenes.},
 bibtype = {article},
 author = {Guédon, Antoine and Monasse, Pascal and Lepetit, Vincent},
 number = {NeurIPS}
}

Next Best View computation (NBV) is a long-standing problem in robotics, and consists in identifying the next most informative sensor position(s) for reconstructing a 3D object or scene efficiently and accurately. Like most current methods, we consider NBV prediction from a depth sensor like Lidar systems. Learning-based methods relying on a volumetric representation of the scene are suitable for path planning, but have lower accuracy than methods using a surface-based representation. However, the latter do not scale well with the size of the scene and constrain the camera to a small number of poses. To obtain the advantages of both representations, we show that we can maximize surface metrics by Monte Carlo integration over a volumetric representation. In particular, we propose an approach, SCONE, that relies on two neural modules: The first module predicts occupancy probability in the entire volume of the scene. Given any new camera pose, the second module samples points in the scene based on their occupancy probability and leverages a self-attention mechanism to predict the visibility of the samples. Finally, we integrate the visibility to evaluate the gain in surface coverage for the new camera pose. NBV is selected as the pose that maximizes the gain in total surface coverage. Our method scales to large scenes and handles free camera motion: It takes as input an arbitrarily large point cloud gathered by a depth sensor as well as camera poses to predict NBV. We demonstrate our approach on a novel dataset made of large and complex 3D scenes.

@inproceedings{
 title = {A Survey on Convolutional Neural Network Accelerators: GPU, FPGA and ASIC},
 type = {inproceedings},
 year = {2022},
 keywords = {ASIC,FPGA,GPU,convolutional neural network,deep learning accelerator},
 pages = {100-107},
 month = {1},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {7},
 city = {Shenzhen, China},
 id = {8a31163f-710d-3cb0-9be5-a9f4c4b9f9f6},
 created = {2023-11-07T09:44:22.169Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:40:02.837Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {73 cites},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {In recent years, artificial intelligence (AI) has been under rapid development, applied in various areas. Among a vast number of neural network (NN) models, the convolutional neural network (CNN) has a mainstream status in application such as image and sound recognition and machine decision. The convolution operation is the most complex and requires acceleration. A practical method is to optimize the architecture of the deep learning processor (DLP). The traditional CPU architecture lacks parallelism and memory bandwidth and is not suitable for CNN operations. Current researches are focused on graphic processing unit (GPU), field programmable gate array (FPGA) and application specific integrated circuit (ASIC). GPU is the maturest and the most widely applied, however it is not flexible and has high cost and energy consumption. Even though FPGA possesses high flexibility and low energy consumption, it is inferior in performance. ASIC, due to targeted design, is advanced in performance and energy consumption. However, it is highly inflexible. This article reviews the research outcomes of the three classic types of processors applied to CNN, and put forward the future research trend. In particular, this paper analyzes and compares the experimental performance of several processors of different types, and then summarizes the respective advantageous application fields. Hence, the novelty of this article is in the summary of practical DLPs, which is expected to provide helps for the AI researchers, and guide the selection of CNN-supporting hardware in industrial application.},
 bibtype = {inproceedings},
 author = {Hu, Yunxiang and Liu, Yuhao and Liu, Zhuovuan},
 doi = {10.1109/ICCRD54409.2022.9730377},
 booktitle = {2022 IEEE 14th International Conference on Computer Research and Development, ICCRD 2022}
}

In recent years, artificial intelligence (AI) has been under rapid development, applied in various areas. Among a vast number of neural network (NN) models, the convolutional neural network (CNN) has a mainstream status in application such as image and sound recognition and machine decision. The convolution operation is the most complex and requires acceleration. A practical method is to optimize the architecture of the deep learning processor (DLP). The traditional CPU architecture lacks parallelism and memory bandwidth and is not suitable for CNN operations. Current researches are focused on graphic processing unit (GPU), field programmable gate array (FPGA) and application specific integrated circuit (ASIC). GPU is the maturest and the most widely applied, however it is not flexible and has high cost and energy consumption. Even though FPGA possesses high flexibility and low energy consumption, it is inferior in performance. ASIC, due to targeted design, is advanced in performance and energy consumption. However, it is highly inflexible. This article reviews the research outcomes of the three classic types of processors applied to CNN, and put forward the future research trend. In particular, this paper analyzes and compares the experimental performance of several processors of different types, and then summarizes the respective advantageous application fields. Hence, the novelty of this article is in the summary of practical DLPs, which is expected to provide helps for the AI researchers, and guide the selection of CNN-supporting hardware in industrial application.

@article{
 title = {Efficient Edge-AI Application Deployment for FPGAs†},
 type = {article},
 year = {2022},
 keywords = {CNN,DNN,FPGA,KV260,Kria,MPSoC,PYNQ,artificial intelligence,deep learning,edge-AI},
 pages = {279},
 volume = {13},
 month = {5},
 publisher = {MDPI},
 day = {28},
 id = {db06f9ab-51f6-3403-b3e7-98fa30442f21},
 created = {2023-11-16T11:41:18.291Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:22:19.081Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {31 cites<br/><br/>Presents multple NN models run on Kria KV260 Xilinx FPGA<br/>15 citations - Greek University of Patras},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Field Programmable Gate Array (FPGA) accelerators have been widely adopted for artificial intelligence (AI) applications on edge devices (Edge-AI) utilizing Deep Neural Networks (DNN) architectures. FPGAs have gained their reputation due to the greater energy efficiency and high parallelism than microcontrollers (MCU) and graphical processing units (GPU), while they are easier to develop and more reconfigurable than the Application Specific Integrated Circuit (ASIC). The development and building of AI applications on resource constraint devices such as FPGAs remains a challenge, however, due to the co-design approach, which requires a valuable expertise in low-level hardware design and in software development. This paper explores the efficacy and the dynamic deployment of hardware accelerated applications on the Kria KV260 development platform based on the Xilinx Kria K26 system-on-module (SoM), which includes a Zynq multiprocessor system-on-chip (MPSoC). The platform supports the Python-based PYNQ framework and maintains a high level of versatility with the support of custom bitstreams (overlays). The demonstration proved the reconfigurabibilty and the overall ease of implementation with low-footprint machine learning (ML) algorithms.},
 bibtype = {article},
 author = {Kalapothas, Stavros and Flamis, Georgios and Kitsos, Paris},
 doi = {10.3390/info13060279},
 journal = {Information (Switzerland)},
 number = {6}
}

Field Programmable Gate Array (FPGA) accelerators have been widely adopted for artificial intelligence (AI) applications on edge devices (Edge-AI) utilizing Deep Neural Networks (DNN) architectures. FPGAs have gained their reputation due to the greater energy efficiency and high parallelism than microcontrollers (MCU) and graphical processing units (GPU), while they are easier to develop and more reconfigurable than the Application Specific Integrated Circuit (ASIC). The development and building of AI applications on resource constraint devices such as FPGAs remains a challenge, however, due to the co-design approach, which requires a valuable expertise in low-level hardware design and in software development. This paper explores the efficacy and the dynamic deployment of hardware accelerated applications on the Kria KV260 development platform based on the Xilinx Kria K26 system-on-module (SoM), which includes a Zynq multiprocessor system-on-chip (MPSoC). The platform supports the Python-based PYNQ framework and maintains a high level of versatility with the support of custom bitstreams (overlays). The demonstration proved the reconfigurabibilty and the overall ease of implementation with low-footprint machine learning (ML) algorithms.

@article{
 title = {Real-time semantic segmentation on FPGAs for autonomous vehicles with hls4ml},
 type = {article},
 year = {2022},
 keywords = {FPGA,autonomous vehicles,computer vision,deep learning,hls4ml,machine learning,semantic segmentation},
 volume = {3},
 month = {11},
 publisher = {Institute of Physics},
 day = {4},
 id = {a5428705-4e91-358a-b683-ee5fb1ade2cc},
 created = {2024-02-07T11:50:23.043Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:38:58.136Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {32 cites<br/><br/>Materials Methods<br/><b>Xilinx ZCU102 </b>evaluation board<br/><b>Cityscapes</b> dataset for segmentation<br/>Efficient Neural Network (<b>ENet</b>) CNN<br/><b>AutoQKeras</b> library<br/>QKeras-to-hls4ml interface for FPGA deployment<br/><br/><b>hls4ml</b> library has an implementation of convolutional layers},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {In this paper, we investigate how field programmable gate arrays can serve as hardware accelerators for real-time semantic segmentation tasks relevant for autonomous driving. Considering compressed versions of the ENet convolutional neural network architecture, we demonstrate a fully-on-chip deployment with a latency of 4.9 ms per image, using less than 30% of the available resources on a Xilinx ZCU102 evaluation board. The latency is reduced to 3 ms per image when increasing the batch size to ten, corresponding to the use case where the autonomous vehicle receives inputs from multiple cameras simultaneously. We show, through aggressive filter reduction and heterogeneous quantization-aware training, and an optimized implementation of convolutional layers, that the power consumption and resource utilization can be significantly reduced while maintaining accuracy on the Cityscapes dataset.},
 bibtype = {article},
 author = {Ghielmetti, Nicolò and Loncar, Vladimir and Pierini, Maurizio and Roed, Marcel and Summers, Sioni and Aarrestad, Thea and Petersson, Christoffer and Linander, Hampus and Ngadiuba, Jennifer and Lin, Kelvin and Harris, Philip},
 doi = {10.1088/2632-2153/ac9cb5},
 journal = {Machine Learning: Science and Technology},
 number = {4}
}

In this paper, we investigate how field programmable gate arrays can serve as hardware accelerators for real-time semantic segmentation tasks relevant for autonomous driving. Considering compressed versions of the ENet convolutional neural network architecture, we demonstrate a fully-on-chip deployment with a latency of 4.9 ms per image, using less than 30% of the available resources on a Xilinx ZCU102 evaluation board. The latency is reduced to 3 ms per image when increasing the batch size to ten, corresponding to the use case where the autonomous vehicle receives inputs from multiple cameras simultaneously. We show, through aggressive filter reduction and heterogeneous quantization-aware training, and an optimized implementation of convolutional layers, that the power consumption and resource utilization can be significantly reduced while maintaining accuracy on the Cityscapes dataset.

@article{
 title = {FPGA-Based Implementation of a CNN Architecture for the On-Board Processing of Very High-Resolution Remote Sensing Images},
 type = {article},
 year = {2022},
 keywords = {Convolutional neural networks (CNNs),deep learning,field-programmable gate array (FPGA),machine learning,remote sensing,target detection},
 pages = {3740-3750},
 volume = {15},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {df35286d-ee41-3868-9d9f-8a6c8089e874},
 created = {2025-01-20T12:09:21.293Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:34:06.477Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {24 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Over the last years, convolutional neural networks (CNNs) have been widely used in remote sensing applications, such as marine surveillance, traffic management, or road networks detection. However, since CNNs have extremely high computational, bandwith, and memory requirements, the hardware implementation of a CNN on space-grade devices like field-programmable gate array (FPGAs) for the on-board processing of the acquired images has brought many challenges, since the computational capabilities of the on-board hardware devices are limited. Hence, implementations have to be carefully planned. In this article, the authors present their work toward the implementation of an efficient CNN onto a space-grade FPGA in order to achieve the on-board processing of very-high resolution remotely sensed images as soon as the data are provided by the sensor. All this work has been conducted within the EU-funded Video Imaging Demonstrator for Earth Observation project. As it will be presented in this article, the work includes the introduction of a methodology based on the project constraints, the evaluation of different state-of-the-art CNN architectures by means of a new efficiency measurement also proposed in this work, the introduction of a new efficient CNN architecture, and finally, its optimized hardware implementation by means of high-level synthesis tools. The results obtained following the proposed methodology demonstrate that the uncovered architecture is able to detect targets of interest in RGB images with a much higher efficiency than state-of-the-art solutions, while requiring a much smaller amount of computing and memory resources.},
 bibtype = {article},
 author = {Neris, Romen and Rodriguez, Adrian and Guerra, Raul and Lopez, Sebastian and Sarmiento, Roberto},
 doi = {10.1109/JSTARS.2022.3169330},
 journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}
}

Over the last years, convolutional neural networks (CNNs) have been widely used in remote sensing applications, such as marine surveillance, traffic management, or road networks detection. However, since CNNs have extremely high computational, bandwith, and memory requirements, the hardware implementation of a CNN on space-grade devices like field-programmable gate array (FPGAs) for the on-board processing of the acquired images has brought many challenges, since the computational capabilities of the on-board hardware devices are limited. Hence, implementations have to be carefully planned. In this article, the authors present their work toward the implementation of an efficient CNN onto a space-grade FPGA in order to achieve the on-board processing of very-high resolution remotely sensed images as soon as the data are provided by the sensor. All this work has been conducted within the EU-funded Video Imaging Demonstrator for Earth Observation project. As it will be presented in this article, the work includes the introduction of a methodology based on the project constraints, the evaluation of different state-of-the-art CNN architectures by means of a new efficiency measurement also proposed in this work, the introduction of a new efficient CNN architecture, and finally, its optimized hardware implementation by means of high-level synthesis tools. The results obtained following the proposed methodology demonstrate that the uncovered architecture is able to detect targets of interest in RGB images with a much higher efficiency than state-of-the-art solutions, while requiring a much smaller amount of computing and memory resources.

@inproceedings{
 title = {Briefly Analysis about CNN Accelerator based on FPGA},
 type = {inproceedings},
 year = {2022},
 keywords = {Acceleration of CNN,Deep learning,FPGA},
 pages = {277-282},
 volume = {202},
 publisher = {Elsevier B.V.},
 id = {8cad37b6-910a-3d5d-b34a-4c1777cda7ad},
 created = {2025-01-20T12:09:24.778Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:21:19.015Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {17 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Since convolutional computation in deep learning is a large and time-consuming computation, researchers often use GPU or FPGA to accelerate these computation. This paper illustrates the advantages of convolutional computation by using FPGA accelerators. In detail, the paper presents some research results of convolutional computation based on FPGA, and explains the current common way of FPGA accelerator design, which uses high-level synthesis and Vitis AI. Finally, we deploy and run YOLOv4 model on the ZCU102 evaluation board using Vitis AI, and perform object detection with the tableware dataset, achieving a recognition result of 96.2%, and 72.5 times higher performance than CPU.},
 bibtype = {inproceedings},
 author = {Wang, Zhichen and Li, Hengyi and Yue, Xuebin and Meng, Lin},
 doi = {10.1016/j.procs.2022.04.036},
 booktitle = {Procedia Computer Science}
}

Since convolutional computation in deep learning is a large and time-consuming computation, researchers often use GPU or FPGA to accelerate these computation. This paper illustrates the advantages of convolutional computation by using FPGA accelerators. In detail, the paper presents some research results of convolutional computation based on FPGA, and explains the current common way of FPGA accelerator design, which uses high-level synthesis and Vitis AI. Finally, we deploy and run YOLOv4 model on the ZCU102 evaluation board using Vitis AI, and perform object detection with the tableware dataset, achieving a recognition result of 96.2%, and 72.5 times higher performance than CPU.

@article{
 title = {Real-time lidar point cloud semantic segmentation for autonomous driving},
 type = {article},
 year = {2022},
 keywords = {CNN,FPGA,GPU,LiDAR,Point cloud,Semantic segmentation},
 volume = {11},
 month = {1},
 publisher = {MDPI},
 day = {1},
 id = {4893ad76-0e73-3387-98e9-3bacbff7f411},
 created = {2025-01-20T13:31:39.628Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:35:51.630Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {35 cites<br/><br/>PCL Semantic Segmentation on FPGA using a NVIDIA Deep<br/>Learning Accelerator (NVDLA) framework on the PL<br/><br/><b>Petalinux </b>image running on Xilinx ZCU104 ARM},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {LiDAR has been widely used in autonomous driving systems to provide high-precision 3D geometric information about the vehicle’s surroundings for perception, localization, and path planning. LiDAR-based point cloud semantic segmentation is an important task with a critical real-time requirement. However, most of the existing convolutional neural network (CNN) models for 3D point cloud semantic segmentation are very complex and can hardly be processed at real-time on an embedded platform. In this study, a lightweight CNN structure was proposed for projection-based LiDAR point cloud semantic segmentation with only 1.9 M parameters that gave an 87% reduction comparing to the state-of-the-art networks. When evaluated on a GPU, the processing time was 38.5 ms per frame, and it achieved a 47.9% mIoU score on Semantic-KITTI dataset. In addition, the proposed CNN is targeted on an FPGA using an NVDLA architecture, which results in a 2.74x speedup over the GPU implementation with a 46 times improvement in terms of power efficiency.},
 bibtype = {article},
 author = {Xie, Xing and Bai, Lin and Huang, Xinming},
 doi = {10.3390/electronics11010011},
 journal = {Electronics (Switzerland)},
 number = {1}
}

LiDAR has been widely used in autonomous driving systems to provide high-precision 3D geometric information about the vehicle’s surroundings for perception, localization, and path planning. LiDAR-based point cloud semantic segmentation is an important task with a critical real-time requirement. However, most of the existing convolutional neural network (CNN) models for 3D point cloud semantic segmentation are very complex and can hardly be processed at real-time on an embedded platform. In this study, a lightweight CNN structure was proposed for projection-based LiDAR point cloud semantic segmentation with only 1.9 M parameters that gave an 87% reduction comparing to the state-of-the-art networks. When evaluated on a GPU, the processing time was 38.5 ms per frame, and it achieved a 47.9% mIoU score on Semantic-KITTI dataset. In addition, the proposed CNN is targeted on an FPGA using an NVDLA architecture, which results in a 2.74x speedup over the GPU implementation with a 46 times improvement in terms of power efficiency.

@inproceedings{
 title = {A Near Sensor Edge Computing System for Point Cloud Semantic Segmentation},
 type = {inproceedings},
 year = {2022},
 pages = {1818-1822},
 volume = {2022-May},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {2e26b083-fa6b-3576-afed-fd3007c86bed},
 created = {2025-04-03T08:01:42.520Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:33:09.728Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {4 cites<br/><br/>PCL pre-processing and segmentation on FPGA DPU on LiDAR data for autonomous vehicles<br/><br/><b>Vitis-AI</b> on Xilinx ZCU102 FPGA},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Point cloud semantic segmentation has attracted attentions due to its robustness to light condition. This makes it an ideal semantic solution for autonomous driving. However, considering the large computation burden and bandwidth demanding of neural networks, putting all the computing into vehicle Electronic Control Unit (ECU) is not efficient or practical. In this paper, we proposed a light weighted point cloud semantic segmentation network based on range view. Due to its simple preprocessing and standard convolution, it is efficient when running on deep learning accelerator like DPU. Furthermore, a near sensor computing system is built for autonomous vehicles. In this system, a FPGA-based deep learning accelerator core (DPU) is placed next to the LiDAR sensor, to perform point cloud preprocessing and segmentation neural network. By leaving only the post-processing step to ECU, this solution heavily alleviate the computation burden of ECU and consequently shortens the decision making and vehicles reaction latency. Our semantic segmentation network achieved 10 frame per second (fps) on Xilinx DPU with computation efficiency 42.5 GOP/W.},
 bibtype = {inproceedings},
 author = {Bai, Lin and Zhao, Yiming and Huang, Xinming},
 doi = {10.1109/ISCAS48785.2022.9937678},
 booktitle = {Proceedings - IEEE International Symposium on Circuits and Systems}
}

Point cloud semantic segmentation has attracted attentions due to its robustness to light condition. This makes it an ideal semantic solution for autonomous driving. However, considering the large computation burden and bandwidth demanding of neural networks, putting all the computing into vehicle Electronic Control Unit (ECU) is not efficient or practical. In this paper, we proposed a light weighted point cloud semantic segmentation network based on range view. Due to its simple preprocessing and standard convolution, it is efficient when running on deep learning accelerator like DPU. Furthermore, a near sensor computing system is built for autonomous vehicles. In this system, a FPGA-based deep learning accelerator core (DPU) is placed next to the LiDAR sensor, to perform point cloud preprocessing and segmentation neural network. By leaving only the post-processing step to ECU, this solution heavily alleviate the computation burden of ECU and consequently shortens the decision making and vehicles reaction latency. Our semantic segmentation network achieved 10 frame per second (fps) on Xilinx DPU with computation efficiency 42.5 GOP/W.

@article{
 title = {FPGA-Based High-Throughput CNN Hardware Accelerator With High Computing Resource Utilization Ratio},
 type = {article},
 year = {2022},
 keywords = {Convolutional neural network,FPGA-based accelerator architecture,high resource utilization ratio,highly efficient storage system},
 pages = {4069-4083},
 volume = {33},
 month = {8},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {b121cecf-6661-30bf-a473-77809512b41d},
 created = {2025-04-07T08:22:56.559Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:23:38.548Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {90 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {The field-programmable gate array (FPGA)-based CNN hardware accelerator adopting single-computing-engine (CE) architecture or multi-CE architecture has attracted great attention in recent years. The actual throughput of the accelerator is also getting higher and higher but is still far below the theoretical throughput due to the inefficient computing resource mapping mechanism and data supply problem, and so on. To solve these problems, a novel composite hardware CNN accelerator architecture is proposed in this article. To perform the convolution layer (CL) efficiently, a novel multiCE architecture based on a row-level pipelined streaming strategy is proposed. For each CE, an optimized mapping mechanism is proposed to improve its computing resource utilization ratio and an efficient data system with continuous data supply is designed to avoid the idle state of the CE. Besides, to relieve the off-chip bandwidth stress, a weight data allocation strategy is proposed. To perform the fully connected layer (FCL), a single-CE architecture based on a batch-based computing method is proposed. Based on these design methods and strategies, visual geometry group network-16 (VGG-16) and ResNet-101 are both implemented on the XC7VX980T FPGA platform. The VGG-16 accelerator consumed 3395 multipliers and got the throughput of 1 TOPS at 150 MHz, that is, about 98.15% of the theoretical throughput (2 × 3395 × 150 MOPS). Similarly, the ResNet-101 accelerator achieved 600 GOPS at 100 MHz, about 96.12% of the theoretical throughput (2 × 3121 × 100 MOPS).},
 bibtype = {article},
 author = {Huang, Wenjin and Wu, Huangtao and Chen, Qingkun and Luo, Conghui and Zeng, Shihao and Li, Tianrui and Huang, Yihua},
 doi = {10.1109/TNNLS.2021.3055814},
 journal = {IEEE Transactions on Neural Networks and Learning Systems},
 number = {8}
}

The field-programmable gate array (FPGA)-based CNN hardware accelerator adopting single-computing-engine (CE) architecture or multi-CE architecture has attracted great attention in recent years. The actual throughput of the accelerator is also getting higher and higher but is still far below the theoretical throughput due to the inefficient computing resource mapping mechanism and data supply problem, and so on. To solve these problems, a novel composite hardware CNN accelerator architecture is proposed in this article. To perform the convolution layer (CL) efficiently, a novel multiCE architecture based on a row-level pipelined streaming strategy is proposed. For each CE, an optimized mapping mechanism is proposed to improve its computing resource utilization ratio and an efficient data system with continuous data supply is designed to avoid the idle state of the CE. Besides, to relieve the off-chip bandwidth stress, a weight data allocation strategy is proposed. To perform the fully connected layer (FCL), a single-CE architecture based on a batch-based computing method is proposed. Based on these design methods and strategies, visual geometry group network-16 (VGG-16) and ResNet-101 are both implemented on the XC7VX980T FPGA platform. The VGG-16 accelerator consumed 3395 multipliers and got the throughput of 1 TOPS at 150 MHz, that is, about 98.15% of the theoretical throughput (2 × 3395 × 150 MOPS). Similarly, the ResNet-101 accelerator achieved 600 GOPS at 100 MHz, about 96.12% of the theoretical throughput (2 × 3121 × 100 MOPS).

@article{
 title = {FPGA-Based CNN for Real-Time UAV Tracking and Detection},
 type = {article},
 year = {2022},
 volume = {3},
 month = {5},
 publisher = {Frontiers Media SA},
 day = {25},
 id = {080ba713-42a2-3d4f-a02f-9ffac4bb01f5},
 created = {2025-04-07T08:22:57.336Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:23:30.993Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {12 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Neural networks (NNs) are now being extensively utilized in various artificial intelligence platforms specifically in the area of image classification and real-time object tracking. We propose a novel design to address the problem of real-time unmanned aerial vehicle (UAV) monitoring and detection using a Zynq UltraScale FPGA-based convolutional neural network (CNN). The biggest challenge while implementing real-time algorithms on FPGAs is the limited DSP hardware resources available on FPGA platforms. Our proposed design overcomes the challenge of autonomous real-time UAV detection and tracking using a Xilinx’s Zynq UltraScale XCZU9EG system on a chip (SoC) platform. Our proposed design explores and provides a solution for overcoming the challenge of limited floating-point resources while maintaining real-time performance. The solution consists of two modules: UAV tracking module and neural network–based UAV detection module. The tracking module uses our novel background-differencing algorithm, while the UAV detection is based on a modified CNN algorithm, designed to give the maximum field-programmable gate array (FPGA) performance. These two modules are designed to complement each other and enabled simultaneously to provide an enhanced real-time UAV detection for any given video input. The proposed system has been tested on real-life flying UAVs, achieving an accuracy of 82%, running at the full frame rate of the input camera for both tracking and neural network (NN) detection, achieving similar performance than an equivalent deep learning processor unit (DPU) with UltraScale FPGA-based HD video and tracking implementation but with lower resource utilization as shown by our results.},
 bibtype = {article},
 author = {Hobden, Peter and Srivastava, Saket and Nurellari, Edmond},
 doi = {10.3389/frspt.2022.878010},
 journal = {Frontiers in Space Technologies}
}

Neural networks (NNs) are now being extensively utilized in various artificial intelligence platforms specifically in the area of image classification and real-time object tracking. We propose a novel design to address the problem of real-time unmanned aerial vehicle (UAV) monitoring and detection using a Zynq UltraScale FPGA-based convolutional neural network (CNN). The biggest challenge while implementing real-time algorithms on FPGAs is the limited DSP hardware resources available on FPGA platforms. Our proposed design overcomes the challenge of autonomous real-time UAV detection and tracking using a Xilinx’s Zynq UltraScale XCZU9EG system on a chip (SoC) platform. Our proposed design explores and provides a solution for overcoming the challenge of limited floating-point resources while maintaining real-time performance. The solution consists of two modules: UAV tracking module and neural network–based UAV detection module. The tracking module uses our novel background-differencing algorithm, while the UAV detection is based on a modified CNN algorithm, designed to give the maximum field-programmable gate array (FPGA) performance. These two modules are designed to complement each other and enabled simultaneously to provide an enhanced real-time UAV detection for any given video input. The proposed system has been tested on real-life flying UAVs, achieving an accuracy of 82%, running at the full frame rate of the input camera for both tracking and neural network (NN) detection, achieving similar performance than an equivalent deep learning processor unit (DPU) with UltraScale FPGA-based HD video and tracking implementation but with lower resource utilization as shown by our results.

@inproceedings{
 title = {An Efficient FPGA Accelerator for Point Cloud},
 type = {inproceedings},
 year = {2022},
 keywords = {Point cloud,hardware architecture,submanifold sparse convolution},
 volume = {2022-September},
 websites = {10.1109/SOCC56010.2022.9908099},
 month = {9},
 publisher = {IEEE Computer Society},
 city = {Belfast, United Kingdom},
 id = {917d3637-eb7e-3116-ae83-2407402c0615},
 created = {2025-04-07T10:01:15.790Z},
 accessed = {2025-04-07},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:33:35.275Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {10 cites<br/><br/>Semantic segmentation on PCL using submanifold sparse<br/>convolutional network (SSCN). <br/>Outperforms GPU in terms of inference time and energy efficiency.<br/><br/><br/><b>Vivado</b> Design Suite on Xilinx ZCU102},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Deep learning-based point cloud processing plays an important role in various vision tasks, such as autonomous driving, virtual reality (VR), and augmented reality (AR). The submanifold sparse convolutional network (SSCN) has been widely used for the point cloud due to its unique advantages in terms of visual results. However, existing convolutional neural network accelerators suffer from non-trivial performance degradation when employed to accelerate SSCN because of the extreme and unstructured sparsity, and the complex computational dependency between the sparsity of the central activation and the neighborhood ones. In this paper, we propose a high performance FPGA-based accelerator for SSCN. Firstly, we develop a zero removing strategy to remove the coarse-grained redundant regions, thus significantly improving computational efficiency. Secondly, we propose a concise encoding scheme to obtain the matching information for efficient point-wise multiplications. Thirdly, we develop a sparse data matching unit and a computing core based on the proposed encoding scheme, which can convert the irregular sparse operations into regular multiply-accumulate operations. Finally, an efficient hardware architecture for the submanifold sparse convolutional layer is developed and implemented on the Xilinx ZCU102 field-programmable gate array board, where the 3D submanifold sparse U-Net is taken as the benchmark. The experimental results demonstrate that our design drastically improves computational efficiency, and can dramatically improve the power efficiency by 51 times compared to GPU.},
 bibtype = {inproceedings},
 author = {Wang, Zilun and Mao, Wendong and Yang, Peixiang and Wang, Zhongfeng and Lin, Jun},
 doi = {10.1109/SOCC56010.2022.9908099},
 booktitle = {International System on Chip Conference}
}

Deep learning-based point cloud processing plays an important role in various vision tasks, such as autonomous driving, virtual reality (VR), and augmented reality (AR). The submanifold sparse convolutional network (SSCN) has been widely used for the point cloud due to its unique advantages in terms of visual results. However, existing convolutional neural network accelerators suffer from non-trivial performance degradation when employed to accelerate SSCN because of the extreme and unstructured sparsity, and the complex computational dependency between the sparsity of the central activation and the neighborhood ones. In this paper, we propose a high performance FPGA-based accelerator for SSCN. Firstly, we develop a zero removing strategy to remove the coarse-grained redundant regions, thus significantly improving computational efficiency. Secondly, we propose a concise encoding scheme to obtain the matching information for efficient point-wise multiplications. Thirdly, we develop a sparse data matching unit and a computing core based on the proposed encoding scheme, which can convert the irregular sparse operations into regular multiply-accumulate operations. Finally, an efficient hardware architecture for the submanifold sparse convolutional layer is developed and implemented on the Xilinx ZCU102 field-programmable gate array board, where the 3D submanifold sparse U-Net is taken as the benchmark. The experimental results demonstrate that our design drastically improves computational efficiency, and can dramatically improve the power efficiency by 51 times compared to GPU.

@inproceedings{
 title = {Convolutional neural network implementations using Vitis AI},
 type = {inproceedings},
 year = {2022},
 keywords = {Convolutional Neural Network (CNN),FPGA,Vitis AI},
 pages = {365-371},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {21262cfd-be57-3a8c-9cf3-1ce1cbfdf1d1},
 created = {2025-04-07T10:41:23.315Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:36:37.939Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {23 cites<br/><br/><b>Paper description</b><br/>Evaluates CNN implementations using Vitis AI.<br/><br/><b>Methods</b>: AI Quantizer for INT8 quantization, AI Compiler, Vitis AI framework, and DPU for FPGA acceleration.<br/><br/><b>Tools</b>: Xilinx Zynq-7000 SoC (XC7Z020), Vitis AI, TensorFlow, Linux.<br/><br/><b>Materials</b>: CIFAR-10 dataset for training and evaluation of CNNs.<br/><br/><b>Architectures</b>: 4-layer, 8-layer, and 84-layer CNNs.<br/><br/><b>Results</b><br/>- Recognition precision after INT8 quantization: 4-layer CNN: 66.6%, 8-layer CNN: 84.9%, 84-layer CNN: 92.5%.<br/>- Power consumption: FPGA is 4.96x more efficient than GPU.<br/>- DPU enables multithreading, improving inference speed.},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Recently, Xilinx has provided a field programmable gate array (FPGA)-based Vitis AI development environment, which is a deep learning framework to accel-erate AI operations and to seek a suitable neural network construction for a target application. We have implemented convolutional neural networks of three types onto the Vitis AI development environment and then we have evaluated their performance, power consumption, design man-hours, and so on. Results confirmed the Vitis AI benefits. Most notably, the FPGA platform power consumption is 4.96 times less than that of a GPU.},
 bibtype = {inproceedings},
 author = {Ushiroyama, Akihiko and Watanabe, Minoru and Watanabe, Nobuya and Nagoya, Akira},
 doi = {10.1109/CCWC54503.2022.9720794},
 booktitle = {2022 IEEE 12th Annual Computing and Communication Workshop and Conference, CCWC 2022}
}

Recently, Xilinx has provided a field programmable gate array (FPGA)-based Vitis AI development environment, which is a deep learning framework to accel-erate AI operations and to seek a suitable neural network construction for a target application. We have implemented convolutional neural networks of three types onto the Vitis AI development environment and then we have evaluated their performance, power consumption, design man-hours, and so on. Results confirmed the Vitis AI benefits. Most notably, the FPGA platform power consumption is 4.96 times less than that of a GPU.

@inproceedings{
 title = {MPSoC4Drones: An Open Framework for ROS2, PX4, and FPGA Integration},
 type = {inproceedings},
 year = {2022},
 pages = {1246-1255},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {5502f383-1916-3533-9764-99efdce8f002},
 created = {2025-04-07T10:49:25.959Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:38:37.287Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {16 cites<br/><br/><b>Paper description</b><br/>MPSoC4Drones: An Open Framework for ROS2, PX4, and FPGA Integration. Proposes a framework for integrating FPGA hardware into drones with PX4 and ROS2 for onboard computation.<br/><br/><b>Methods</b>: FPGA hardware acceleration using Zynq Ultra96-V2, integration with ROS2 and PX4.<br/><br/><b>Tools</b>: Xilinx Vivado, PetaLinux, Ubuntu 20.04, ROS2 Foxy, PX4 autopilot.<br/><br/><b>Materials</b>: Ultra96-V2 board.<br/><br/><b>Results</b><br/>- Achieved 25 FPS for a deeplearning-based person detection.<br/>- Power consumption: 6.4 W peak, significantly lower than GPU.},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Autonomous drones are facing a tough efficiency challenge due to limitations on the utilized processing hardware units. Among these limitations is the tradeoff between fast computing and low power consumption; between functional complexity and flight time. Recent progressions point to FPGAs for accelerating heavy processing. In this work, we present the MPSoC4Drones Framework; a novel framework for organizing FPGA-design and OS build projects. The framework combines tools for creating bootable images for the Ultra96-V2 board.We show how MPSoC4Drones organizes the build, combining the latest well-known tools for research and industrial drone development, Ubuntu 20.04, PX4 autopilot, and ROS2 middleware. We validate the framework through a computationally intensive deep learning use case implemented in the MPSoC4Drones framework. We show the superior throughput and low power consumption of FPGA processing compared to classical CPU and GPU approaches. Finally, we offer the full framework as open-source.},
 bibtype = {inproceedings},
 author = {Nyboe, Frederik Falk and Malle, Nicolaj Haarhoj and Ebeid, Emad},
 doi = {10.1109/ICUAS54217.2022.9836055},
 booktitle = {2022 International Conference on Unmanned Aircraft Systems, ICUAS 2022}
}

Autonomous drones are facing a tough efficiency challenge due to limitations on the utilized processing hardware units. Among these limitations is the tradeoff between fast computing and low power consumption; between functional complexity and flight time. Recent progressions point to FPGAs for accelerating heavy processing. In this work, we present the MPSoC4Drones Framework; a novel framework for organizing FPGA-design and OS build projects. The framework combines tools for creating bootable images for the Ultra96-V2 board.We show how MPSoC4Drones organizes the build, combining the latest well-known tools for research and industrial drone development, Ubuntu 20.04, PX4 autopilot, and ROS2 middleware. We validate the framework through a computationally intensive deep learning use case implemented in the MPSoC4Drones framework. We show the superior throughput and low power consumption of FPGA processing compared to classical CPU and GPU approaches. Finally, we offer the full framework as open-source.

@inproceedings{
 title = {HDL Design and Implementation of a Convolutional Neural Network for Efficient FPGA/Hardware Platforms},
 type = {inproceedings},
 year = {2022},
 keywords = {CNNs,Hardware Implementation,Machine Learning,ReLU},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {fdf6209a-69f7-39f1-9217-67b6e447cbf5},
 created = {2025-04-07T14:12:59.287Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T15:04:17.157Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {1 cites},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Neural networks are a set of algorithms modeled on the human brain, primarily for pattern recognition which has a special category called convolutional neural networks. Convolutional neural networks also called as ConvNets or CNNs which is the most important tool for the machine learning practitioners today. CNN has proved that its works effectively in various areas such as recognition of image and classification. This CNNs also have been proved successful in identifying objects and faces and identifying traffic lights apart from powering vision in robots and self-driving cars. Among these layers hidden layer plays an important role which inturn consists of convolution, Activation Filter (ReLU) and pooling. Any CNN which is used in identifying image or classifying an object etc., must contain these blocks to work efficiently for classification of the higher representation of the image content. The primary contribution of this work is a step towards assisting an introductory designer targeting to design and implement a basic CNN in FPGA and detailed design procedure and steps are outlined in the paper. Basic CNN by taking 5*5 input feature is designed in Xilinx Vivado and parameters such as area, delay and speed are compared.},
 bibtype = {inproceedings},
 author = {Lahari, P. L. and Yellampalli, Siva Sankar and Vaddi, Ramesh},
 doi = {10.1109/I4Tech55392.2022.9952465},
 booktitle = {2022 International Conference on Industry 4.0 Technology, I4Tech 2022}
}

Neural networks are a set of algorithms modeled on the human brain, primarily for pattern recognition which has a special category called convolutional neural networks. Convolutional neural networks also called as ConvNets or CNNs which is the most important tool for the machine learning practitioners today. CNN has proved that its works effectively in various areas such as recognition of image and classification. This CNNs also have been proved successful in identifying objects and faces and identifying traffic lights apart from powering vision in robots and self-driving cars. Among these layers hidden layer plays an important role which inturn consists of convolution, Activation Filter (ReLU) and pooling. Any CNN which is used in identifying image or classifying an object etc., must contain these blocks to work efficiently for classification of the higher representation of the image content. The primary contribution of this work is a step towards assisting an introductory designer targeting to design and implement a basic CNN in FPGA and detailed design procedure and steps are outlined in the paper. Basic CNN by taking 5*5 input feature is designed in Xilinx Vivado and parameters such as area, delay and speed are compared.

@article{
 title = {Chaos LiDAR Based RGB-D Face Classification System With Embedded CNN Accelerator on FPGAs},
 type = {article},
 year = {2022},
 keywords = {Chaos LiDAR,RGB-D,block-based embedded CNN (eCNN),correlator processor,depth construction,face classification,field programmable gate arrays (FPGA)},
 pages = {4847-4859},
 volume = {69},
 month = {12},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {6fe1810d-7217-39a8-a1ad-4cfc489d110a},
 created = {2025-06-03T16:11:31.633Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:34:40.899Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {18 cites},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Face classification is important in many applications such as surveillance, border control, and security systems. However, wide variations in environments such as insufficient light, large distances or pose angles make the task challenging. Depth sensors are added with RGB cameras for improving classification accuracy but commercial RGB-D sensors are most targeted for indoors applications. In this paper, we present and design a Chaos LiDAR depth sesnor that provides high-precision depth images through intelligent correlation processing for both indoors and outdoors applications. Our Chaos LiDAR depth sensor detects range from 2 to 40 meters with precision around 8mm at 20-meter. With the Chaos LiDAR depth as input, we design a RGB-D based face classification embedded CNN (eCNN) model for wide range applications such as dim illumination, various distances and large poses. Our Chaos LiDAR increases around 14.27% classification accuracy compared to RealSense D435i for distance from 3 to 5 meter. The eCNN face classification subsystem is implemented in Xilinx ZCU 102 and achieves 11.11 ms inference time. The eCNN engine achieves a peak throughput at 614.4 GOPS. The overall system including Chaos LiDAR, correlation and eCNN FPGA achieves face classification inference rate of 10fps.},
 bibtype = {article},
 author = {Chiu, Ching Te and Ding, Yu Chun and Lin, Wei Chen and Chen, Wei Jyun and Wu, Shu Yun and Huang, Chao Tsung and Lin, Chun Yeh and Chang, Chia Yu and Lee, Meng Jui and Tatsunori, Shimazu and Chen, Tsung and Lin, Fan Yi and Huang, Yuan Hao},
 doi = {10.1109/TCSI.2022.3190430},
 journal = {IEEE Transactions on Circuits and Systems I: Regular Papers},
 number = {12}
}

Face classification is important in many applications such as surveillance, border control, and security systems. However, wide variations in environments such as insufficient light, large distances or pose angles make the task challenging. Depth sensors are added with RGB cameras for improving classification accuracy but commercial RGB-D sensors are most targeted for indoors applications. In this paper, we present and design a Chaos LiDAR depth sesnor that provides high-precision depth images through intelligent correlation processing for both indoors and outdoors applications. Our Chaos LiDAR depth sensor detects range from 2 to 40 meters with precision around 8mm at 20-meter. With the Chaos LiDAR depth as input, we design a RGB-D based face classification embedded CNN (eCNN) model for wide range applications such as dim illumination, various distances and large poses. Our Chaos LiDAR increases around 14.27% classification accuracy compared to RealSense D435i for distance from 3 to 5 meter. The eCNN face classification subsystem is implemented in Xilinx ZCU 102 and achieves 11.11 ms inference time. The eCNN engine achieves a peak throughput at 614.4 GOPS. The overall system including Chaos LiDAR, correlation and eCNN FPGA achieves face classification inference rate of 10fps.

@article{
 title = {State of Art IoT and Edge Embedded Systems for Real-Time Machine Vision Applications},
 type = {article},
 year = {2022},
 keywords = {ASIC,Edge machine vision systems,FPGA,GPU,IoT,Multicore CPU},
 pages = {58287-58301},
 volume = {10},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {13678d6d-c131-3e06-9554-e6b268327276},
 created = {2025-06-04T14:17:52.686Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:41:06.129Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {26 cites},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc},
 private_publication = {false},
 abstract = {IoT and edge devices dedicated to run machine vision algorithms are usually few years lagging currently available state-of-the-art technologies for hardware accelerators. This is mainly due to the non-negligible time delay required to implement and assess related algorithms. Among possible hardware platforms which are potentially being explored to handle real-time machine vision tasks, multi-core CPU and Graphical Processing Unit (GPU) platforms remain the most widely used ones over Field Programmable Gate Array (FPGA) and Application Specific Integrated Circuit (ASIC)-based platforms. This is mainly due to the availability of powerful and user friendly software development tools, in addition to their lower cost, and obviously their high computation power with reasonable form factor and power consumption. Nevertheless, the trend now is towards a System-On-Chip (SOC) processors which combine ASIC/FPGA accelerators with GPU/multicore CPUs. This paper presents different state of the art IoT and edge machine vision technologies along with their performance and limitations. It can be a good reference for researchers involved in designing state of the art IoT embedded systems for machine vision applications.},
 bibtype = {article},
 author = {Meribout, Mahmoud and Baobaid, Asma and Khaoua, Mohammed Ould and Tiwari, Varun Kumar and Pena, Juan Pablo},
 doi = {10.1109/ACCESS.2022.3175496},
 journal = {IEEE Access}
}

IoT and edge devices dedicated to run machine vision algorithms are usually few years lagging currently available state-of-the-art technologies for hardware accelerators. This is mainly due to the non-negligible time delay required to implement and assess related algorithms. Among possible hardware platforms which are potentially being explored to handle real-time machine vision tasks, multi-core CPU and Graphical Processing Unit (GPU) platforms remain the most widely used ones over Field Programmable Gate Array (FPGA) and Application Specific Integrated Circuit (ASIC)-based platforms. This is mainly due to the availability of powerful and user friendly software development tools, in addition to their lower cost, and obviously their high computation power with reasonable form factor and power consumption. Nevertheless, the trend now is towards a System-On-Chip (SOC) processors which combine ASIC/FPGA accelerators with GPU/multicore CPUs. This paper presents different state of the art IoT and edge machine vision technologies along with their performance and limitations. It can be a good reference for researchers involved in designing state of the art IoT embedded systems for machine vision applications.

@article{
 title = {Estimation of 2D Bounding Box Orientation with Convex-Hull Points - A Quantitative Evaluation on Accuracy and Efficiency},
 type = {article},
 year = {2021},
 pages = {945-950},
 id = {ae23b22a-c103-3616-800c-6dadaab5d98a},
 created = {2021-01-28T12:21:22.357Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-01-29T13:17:54.524Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {471f331d-8560-4b9e-b910-e5f849b6fcfd},
 private_publication = {false},
 bibtype = {article},
 author = {Liu, Yang and Liu, Bingbing and Zhang, Hongbo},
 doi = {10.1109/iv47402.2020.9304788},
 number = {Iv}
}

@article{
 title = {PV-RCNN++: Point-Voxel Feature Set Abstraction With Local Vector Representation for 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {1-17},
 websites = {http://arxiv.org/abs/2102.00463},
 id = {f8ad32e7-f771-39cc-a534-0e2ec29e08c4},
 created = {2021-03-04T15:41:23.595Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-24T15:42:26.064Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Shi2021},
 folder_uuids = {bc1835e2-32e3-4f2a-b03c-9540bbbd02e0},
 private_publication = {false},
 abstract = {3D object detection is receiving increasing attention from both industry and academia thanks to its wide applications in various fields. In this paper, we propose the Point-Voxel Region based Convolution Neural Networks (PV-RCNNs) for accurate 3D detection from point clouds. First, we propose a novel 3D object detector, PV-RCNN-v1, which employs the voxel-to-keypoint scene encoding and keypoint-to-grid RoI feature abstraction two novel steps. These two steps deeply incorporate both 3D voxel CNN and PointNet-based set abstraction for learning discriminative point-cloud features. Second, we propose a more advanced framework, PV-RCNN-v2, for more efficient and accurate 3D detection. It consists of two major improvements, where the first one is the sectorized proposal-centric strategy for efficiently producing more representative and uniformly distributed keypoints, and the second one is the VectorPool aggregation to replace set abstraction for better aggregating local point-cloud features with much less resource consumption. With these two major modifications, our PV-RCNN-v2 runs more than twice as fast as the v1 version while still achieving better performance on the large-scale Waymo Open Dataset with 150m * 150m detection range. Extensive experiments demonstrate that our proposed PV-RCNNs significantly outperform previous state-of-the-art 3D detection methods on both the Waymo Open Dataset and the highly-competitive KITTI benchmark.},
 bibtype = {article},
 author = {Shi, Shaoshuai and Jiang, Li and Deng, Jiajun and Wang, Zhe and Guo, Chaoxu and Shi, Jianping and Wang, Xiaogang and Li, Hongsheng}
}

3D object detection is receiving increasing attention from both industry and academia thanks to its wide applications in various fields. In this paper, we propose the Point-Voxel Region based Convolution Neural Networks (PV-RCNNs) for accurate 3D detection from point clouds. First, we propose a novel 3D object detector, PV-RCNN-v1, which employs the voxel-to-keypoint scene encoding and keypoint-to-grid RoI feature abstraction two novel steps. These two steps deeply incorporate both 3D voxel CNN and PointNet-based set abstraction for learning discriminative point-cloud features. Second, we propose a more advanced framework, PV-RCNN-v2, for more efficient and accurate 3D detection. It consists of two major improvements, where the first one is the sectorized proposal-centric strategy for efficiently producing more representative and uniformly distributed keypoints, and the second one is the VectorPool aggregation to replace set abstraction for better aggregating local point-cloud features with much less resource consumption. With these two major modifications, our PV-RCNN-v2 runs more than twice as fast as the v1 version while still achieving better performance on the large-scale Waymo Open Dataset with 150m * 150m detection range. Extensive experiments demonstrate that our proposed PV-RCNNs significantly outperform previous state-of-the-art 3D detection methods on both the Waymo Open Dataset and the highly-competitive KITTI benchmark.

@article{
 title = {FPS-Net: A Convolutional Fusion Network for Large-Scale LiDAR Point Cloud Segmentation},
 type = {article},
 year = {2021},
 keywords = {autonomous driving,lidar,point cloud,scene,semantic segmentation,spherical projection},
 websites = {http://arxiv.org/abs/2103.00738},
 id = {18de3f0a-63c2-3512-a357-6acaf1f5ecb9},
 created = {2021-03-04T15:41:23.623Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-03-06T08:50:44.516Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {bc1835e2-32e3-4f2a-b03c-9540bbbd02e0},
 private_publication = {false},
 abstract = {Scene understanding based on LiDAR point cloud is an essential task for autonomous cars to drive safely, which often employs spherical projection to map 3D point cloud into multi-channel 2D images for semantic segmentation. Most existing methods simply stack different point attributes/modalities (e.g. coordinates, intensity, depth, etc.) as image channels to increase information capacity, but ignore distinct characteristics of point attributes in different image channels. We design FPS-Net, a convolutional fusion network that exploits the uniqueness and discrepancy among the projected image channels for optimal point cloud segmentation. FPS-Net adopts an encoder-decoder structure. Instead of simply stacking multiple channel images as a single input, we group them into different modalities to first learn modality-specific features separately and then map the learned features into a common high-dimensional feature space for pixel-level fusion and learning. Specifically, we design a residual dense block with multiple receptive fields as a building block in the encoder which preserves detailed information in each modality and learns hierarchical modality-specific and fused features effectively. In the FPS-Net decoder, we use a recurrent convolution block likewise to hierarchically decode fused features into output space for pixel-level classification. Extensive experiments conducted on two widely adopted point cloud datasets show that FPS-Net achieves superior semantic segmentation as compared with state-of-the-art projection-based methods. In addition, the proposed modality fusion idea is compatible with typical projection-based methods and can be incorporated into them with consistent performance improvements.},
 bibtype = {article},
 author = {Xiao, Aoran and Yang, Xiaofei and Lu, Shijian and Guan, Dayan and Huang, Jiaxing}
}

Scene understanding based on LiDAR point cloud is an essential task for autonomous cars to drive safely, which often employs spherical projection to map 3D point cloud into multi-channel 2D images for semantic segmentation. Most existing methods simply stack different point attributes/modalities (e.g. coordinates, intensity, depth, etc.) as image channels to increase information capacity, but ignore distinct characteristics of point attributes in different image channels. We design FPS-Net, a convolutional fusion network that exploits the uniqueness and discrepancy among the projected image channels for optimal point cloud segmentation. FPS-Net adopts an encoder-decoder structure. Instead of simply stacking multiple channel images as a single input, we group them into different modalities to first learn modality-specific features separately and then map the learned features into a common high-dimensional feature space for pixel-level fusion and learning. Specifically, we design a residual dense block with multiple receptive fields as a building block in the encoder which preserves detailed information in each modality and learns hierarchical modality-specific and fused features effectively. In the FPS-Net decoder, we use a recurrent convolution block likewise to hierarchically decode fused features into output space for pixel-level classification. Extensive experiments conducted on two widely adopted point cloud datasets show that FPS-Net achieves superior semantic segmentation as compared with state-of-the-art projection-based methods. In addition, the proposed modality fusion idea is compatible with typical projection-based methods and can be incorporated into them with consistent performance improvements.

@article{
 title = {Pruning and Quantization for Deep Neural Network Acceleration: A Survey},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2101.09671},
 month = {1},
 day = {24},
 id = {72da8552-3a9f-3839-9b1d-4cdf5aff002f},
 created = {2021-06-14T08:43:51.909Z},
 accessed = {2021-06-14},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-06-14T08:44:38.180Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {c9e2a751-ce83-45dd-9c0e-bdac57df3cf4},
 private_publication = {false},
 abstract = {Deep neural networks have been applied in many applications exhibiting extraordinary abilities in the field of computer vision. However, complex network architectures challenge efficient real-time deployment and require significant computation resources and energy costs. These challenges can be overcome through optimizations such as network compression. Network compression can often be realized with little loss of accuracy. In some cases accuracy may even improve. This paper provides a survey on two types of network compression: pruning and quantization. Pruning can be categorized as static if it is performed offline or dynamic if it is performed at run-time. We compare pruning techniques and describe criteria used to remove redundant computations. We discuss trade-offs in element-wise, channel-wise, shape-wise, filter-wise, layer-wise and even network-wise pruning. Quantization reduces computations by reducing the precision of the datatype. Weights, biases, and activations may be quantized typically to 8-bit integers although lower bit width implementations are also discussed including binary neural networks. Both pruning and quantization can be used independently or combined. We compare current techniques, analyze their strengths and weaknesses, present compressed network accuracy results on a number of frameworks, and provide practical guidance for compressing networks.},
 bibtype = {article},
 author = {Liang, Tailin and Glossner, John and Wang, Lei and Shi, Shaobo}
}

Deep neural networks have been applied in many applications exhibiting extraordinary abilities in the field of computer vision. However, complex network architectures challenge efficient real-time deployment and require significant computation resources and energy costs. These challenges can be overcome through optimizations such as network compression. Network compression can often be realized with little loss of accuracy. In some cases accuracy may even improve. This paper provides a survey on two types of network compression: pruning and quantization. Pruning can be categorized as static if it is performed offline or dynamic if it is performed at run-time. We compare pruning techniques and describe criteria used to remove redundant computations. We discuss trade-offs in element-wise, channel-wise, shape-wise, filter-wise, layer-wise and even network-wise pruning. Quantization reduces computations by reducing the precision of the datatype. Weights, biases, and activations may be quantized typically to 8-bit integers although lower bit width implementations are also discussed including binary neural networks. Both pruning and quantization can be used independently or combined. We compare current techniques, analyze their strengths and weaknesses, present compressed network accuracy results on a number of frameworks, and provide practical guidance for compressing networks.

@article{
 title = {Point Cloud Learning with Transformer},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.13636},
 id = {efbfd798-a585-3431-aa1b-a8b5f4961381},
 created = {2021-06-21T08:44:26.178Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-12T06:29:25.133Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {70eb910f-9399-46d8-a4d0-ade5435237b7,597192a3-7679-4832-a554-980990d8ac9b,d54ba66b-a8cf-41de-8e2d-c3256f322e07},
 private_publication = {false},
 abstract = {Remarkable performance from Transformer networks in Natural Language Processing promote the development of these models in dealing with computer vision tasks such as image recognition and segmentation. In this paper, we introduce a novel framework, called Multi-level Multi-scale Point Transformer (MLMSPT) that works directly on the irregular point clouds for representation learning. Specifically, a point pyramid transformer is investigated to model features with diverse resolutions or scales we defined, followed by a multi-level transformer module to aggregate contextual information from different levels of each scale and enhance their interactions. While a multi-scale transformer module is designed to capture the dependencies among representations across different scales. Extensive evaluation on public benchmark datasets demonstrate the effectiveness and the competitive performance of our methods on 3D shape classification, part segmentation and semantic segmentation tasks.},
 bibtype = {article},
 author = {Han, Xian-Feng and Kuang, Yu-Jia and Xiao, Guo-Qiang},
 number = {3}
}

Remarkable performance from Transformer networks in Natural Language Processing promote the development of these models in dealing with computer vision tasks such as image recognition and segmentation. In this paper, we introduce a novel framework, called Multi-level Multi-scale Point Transformer (MLMSPT) that works directly on the irregular point clouds for representation learning. Specifically, a point pyramid transformer is investigated to model features with diverse resolutions or scales we defined, followed by a multi-level transformer module to aggregate contextual information from different levels of each scale and enhance their interactions. While a multi-scale transformer module is designed to capture the dependencies among representations across different scales. Extensive evaluation on public benchmark datasets demonstrate the effectiveness and the competitive performance of our methods on 3D shape classification, part segmentation and semantic segmentation tasks.

@article{
 title = {PCT: Point cloud transformer},
 type = {article},
 year = {2021},
 keywords = {3D computer vision,Transformer,deep learning,point cloud processing},
 pages = {187-199},
 volume = {7},
 id = {a5d79181-17da-3f52-bb47-220699339308},
 created = {2021-06-21T08:44:26.372Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-15T11:24:42.580Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {70eb910f-9399-46d8-a4d0-ade5435237b7,597192a3-7679-4832-a554-980990d8ac9b,07e07de9-bcac-4934-a82b-d0aff540e56d,11276190-b8fe-4c3a-a42f-f604438ad4db,d54ba66b-a8cf-41de-8e2d-c3256f322e07},
 private_publication = {false},
 abstract = {The irregular domain and lack of ordering make it challenging to design deep neural networks for point cloud processing. This paper presents a novel framework named Point Cloud Transformer (PCT) for point cloud learning. PCT is based on Transformer, which achieves huge success in natural language processing and displays great potential in image processing. It is inherently permutation invariant for processing a sequence of points, making it well-suited for point cloud learning. To better capture local context within the point cloud, we enhance input embedding with the support of farthest point sampling and nearest neighbor search. Extensive experiments demonstrate that the PCT achieves the state-of-the-art performance on shape classification, part segmentation, semantic segmentation, and normal estimation tasks.},
 bibtype = {article},
 author = {Guo, Meng Hao and Cai, Jun Xiong and Liu, Zheng Ning and Mu, Tai Jiang and Martin, Ralph R. and Hu, Shi Min},
 doi = {10.1007/s41095-021-0229-5},
 journal = {Computational Visual Media},
 number = {2}
}

The irregular domain and lack of ordering make it challenging to design deep neural networks for point cloud processing. This paper presents a novel framework named Point Cloud Transformer (PCT) for point cloud learning. PCT is based on Transformer, which achieves huge success in natural language processing and displays great potential in image processing. It is inherently permutation invariant for processing a sequence of points, making it well-suited for point cloud learning. To better capture local context within the point cloud, we enhance input embedding with the support of farthest point sampling and nearest neighbor search. Extensive experiments demonstrate that the PCT achieves the state-of-the-art performance on shape classification, part segmentation, semantic segmentation, and normal estimation tasks.

@article{
 title = {Geometric Deep Learning: Grids, Groups, Graphs, Geodesics, and Gauges},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.13478},
 id = {1e903e3b-d888-3d47-9a2a-2c6cfb04c4e9},
 created = {2021-07-12T09:40:52.521Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-01T13:11:54.912Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {368c6572-df92-4840-8400-80e7c9ee2dd7,20ccb950-fef9-4ee1-800c-a60ba9f1df16},
 private_publication = {false},
 abstract = {The last decade has witnessed an experimental revolution in data science and machine learning, epitomised by deep learning methods. Indeed, many high-dimensional learning tasks previously thought to be beyond reach -- such as computer vision, playing Go, or protein folding -- are in fact feasible with appropriate computational scale. Remarkably, the essence of deep learning is built from two simple algorithmic principles: first, the notion of representation or feature learning, whereby adapted, often hierarchical, features capture the appropriate notion of regularity for each task, and second, learning by local gradient-descent type methods, typically implemented as backpropagation. While learning generic functions in high dimensions is a cursed estimation problem, most tasks of interest are not generic, and come with essential pre-defined regularities arising from the underlying low-dimensionality and structure of the physical world. This text is concerned with exposing these regularities through unified geometric principles that can be applied throughout a wide spectrum of applications. Such a 'geometric unification' endeavour, in the spirit of Felix Klein's Erlangen Program, serves a dual purpose: on one hand, it provides a common mathematical framework to study the most successful neural network architectures, such as CNNs, RNNs, GNNs, and Transformers. On the other hand, it gives a constructive procedure to incorporate prior physical knowledge into neural architectures and provide principled way to build future architectures yet to be invented.},
 bibtype = {article},
 author = {Bronstein, Michael M. and Bruna, Joan and Cohen, Taco and Veličković, Petar}
}

The last decade has witnessed an experimental revolution in data science and machine learning, epitomised by deep learning methods. Indeed, many high-dimensional learning tasks previously thought to be beyond reach -- such as computer vision, playing Go, or protein folding -- are in fact feasible with appropriate computational scale. Remarkably, the essence of deep learning is built from two simple algorithmic principles: first, the notion of representation or feature learning, whereby adapted, often hierarchical, features capture the appropriate notion of regularity for each task, and second, learning by local gradient-descent type methods, typically implemented as backpropagation. While learning generic functions in high dimensions is a cursed estimation problem, most tasks of interest are not generic, and come with essential pre-defined regularities arising from the underlying low-dimensionality and structure of the physical world. This text is concerned with exposing these regularities through unified geometric principles that can be applied throughout a wide spectrum of applications. Such a 'geometric unification' endeavour, in the spirit of Felix Klein's Erlangen Program, serves a dual purpose: on one hand, it provides a common mathematical framework to study the most successful neural network architectures, such as CNNs, RNNs, GNNs, and Transformers. On the other hand, it gives a constructive procedure to incorporate prior physical knowledge into neural architectures and provide principled way to build future architectures yet to be invented.

@article{
 title = {Random Features Strengthen Graph Neural Networks},
 type = {article},
 year = {2021},
 pages = {333-341},
 id = {80b59a54-c323-3d62-b01f-b43ca732e4ab},
 created = {2021-07-12T10:19:36.603Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-12T10:19:56.925Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {20ccb950-fef9-4ee1-800c-a60ba9f1df16},
 private_publication = {false},
 abstract = {Graph neural networks (GNNs) are powerful machine learning models for various graph learning tasks. Recently, the limitations of the expressive power of various GNN models have been revealed. For example, GNNs cannot distinguish some non-isomorphic graphs and they cannot learn efficient graph algorithms. In this paper, we demonstrate that GNNs become powerful just by adding a random feature to each node. We prove that the random features enable GNNs to learn almost optimal polynomial-time approximation algorithms for the minimum dominating set problem and maximum matching problem in terms of approximation ratios. The main advantage of our method is that it can be combined with off-the-shelf GNN models with slight modifications. Through experiments, we show that the addition of random features enables GNNs to solve various problems that normal GNNs, including the graph convolutional networks (GCNs) and graph isomorphism networks (GINs), cannot solve.},
 bibtype = {article},
 author = {Sato, Ryoma and Yamada, Makoto and Kashima, Hisashi},
 doi = {10.1137/1.9781611976700.38},
 journal = {Proceedings of the 2021 SIAM International Conference on Data Mining (SDM)}
}

Graph neural networks (GNNs) are powerful machine learning models for various graph learning tasks. Recently, the limitations of the expressive power of various GNN models have been revealed. For example, GNNs cannot distinguish some non-isomorphic graphs and they cannot learn efficient graph algorithms. In this paper, we demonstrate that GNNs become powerful just by adding a random feature to each node. We prove that the random features enable GNNs to learn almost optimal polynomial-time approximation algorithms for the minimum dominating set problem and maximum matching problem in terms of approximation ratios. The main advantage of our method is that it can be combined with off-the-shelf GNN models with slight modifications. Through experiments, we show that the addition of random features enables GNNs to solve various problems that normal GNNs, including the graph convolutional networks (GCNs) and graph isomorphism networks (GINs), cannot solve.

@article{
 title = {Spectral Normalisation for Deep Reinforcement Learning: an Optimisation Perspective},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.05246},
 id = {01e75554-b7ad-3ce9-bd78-1b4aad8f581e},
 created = {2021-07-12T14:15:35.205Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-12T14:17:09.133Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {85ed9c29-c272-40dc-a01a-f912101de83a},
 private_publication = {false},
 abstract = {Most of the recent deep reinforcement learning advances take an RL-centric perspective and focus on refinements of the training objective. We diverge from this view and show we can recover the performance of these developments not by changing the objective, but by regularising the value-function estimator. Constraining the Lipschitz constant of a single layer using spectral normalisation is sufficient to elevate the performance of a Categorical-DQN agent to that of a more elaborated \rainbow agent on the challenging Atari domain. We conduct ablation studies to disentangle the various effects normalisation has on the learning dynamics and show that is sufficient to modulate the parameter updates to recover most of the performance of spectral normalisation. These findings hint towards the need to also focus on the neural component and its learning dynamics to tackle the peculiarities of Deep Reinforcement Learning.},
 bibtype = {article},
 author = {Gogianu, Florin and Berariu, Tudor and Rosca, Mihaela and Clopath, Claudia and Busoniu, Lucian and Pascanu, Razvan}
}

Most of the recent deep reinforcement learning advances take an RL-centric perspective and focus on refinements of the training objective. We diverge from this view and show we can recover the performance of these developments not by changing the objective, but by regularising the value-function estimator. Constraining the Lipschitz constant of a single layer using spectral normalisation is sufficient to elevate the performance of a Categorical-DQN agent to that of a more elaborated \rainbow agent on the challenging Atari domain. We conduct ablation studies to disentangle the various effects normalisation has on the learning dynamics and show that is sufficient to modulate the parameter updates to recover most of the performance of spectral normalisation. These findings hint towards the need to also focus on the neural component and its learning dynamics to tackle the peculiarities of Deep Reinforcement Learning.

@article{
 title = {Drawing Multiple Augmentation Samples Per Image During Training Efficiently Decreases Test Error},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.13343},
 id = {f7215b77-2673-3979-9ac2-c114104553b5},
 created = {2021-07-12T14:15:35.999Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-12T14:17:00.515Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {85ed9c29-c272-40dc-a01a-f912101de83a},
 private_publication = {false},
 abstract = {In computer vision, it is standard practice to draw a single sample from the data augmentation procedure for each unique image in the mini-batch, however it is not clear whether this choice is optimal for generalization. In this work, we provide a detailed empirical evaluation of how the number of augmentation samples per unique image influences performance on held out data. Remarkably, we find that drawing multiple samples per image consistently enhances the test accuracy achieved for both small and large batch training, despite reducing the number of unique training examples in each mini-batch. This benefit arises even when different augmentation multiplicities perform the same number of parameter updates and gradient evaluations. Our results suggest that, although the variance in the gradient estimate arising from subsampling the dataset has an implicit regularization benefit, the variance which arises from the data augmentation process harms test accuracy. By applying augmentation multiplicity to the recently proposed NFNet model family, we achieve a new ImageNet state of the art of 86.8$\%$ top-1 w/o extra data.},
 bibtype = {article},
 author = {Fort, Stanislav and Brock, Andrew and Pascanu, Razvan and De, Soham and Smith, Samuel L.}
}

In computer vision, it is standard practice to draw a single sample from the data augmentation procedure for each unique image in the mini-batch, however it is not clear whether this choice is optimal for generalization. In this work, we provide a detailed empirical evaluation of how the number of augmentation samples per unique image influences performance on held out data. Remarkably, we find that drawing multiple samples per image consistently enhances the test accuracy achieved for both small and large batch training, despite reducing the number of unique training examples in each mini-batch. This benefit arises even when different augmentation multiplicities perform the same number of parameter updates and gradient evaluations. Our results suggest that, although the variance in the gradient estimate arising from subsampling the dataset has an implicit regularization benefit, the variance which arises from the data augmentation process harms test accuracy. By applying augmentation multiplicity to the recently proposed NFNet model family, we achieve a new ImageNet state of the art of 86.8$\%$ top-1 w/o extra data.

@article{
 title = {Transformers for Computer Vision},
 type = {article},
 year = {2021},
 id = {ecca39fc-9a23-3448-8923-8baec26a709e},
 created = {2021-07-19T10:48:23.785Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-19T10:48:52.671Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {40817151-8323-4487-8c1e-fe067729f714},
 private_publication = {false},
 bibtype = {article},
 author = {Dosovitskiy, Alexey}
}

@article{
 title = {Tutorial on Variational Autoencoders Why are VAEs interesting ?},
 type = {article},
 year = {2021},
 id = {0fdbf08f-0e70-32a9-a4df-427e0bc1e959},
 created = {2021-07-19T10:48:23.904Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-19T10:49:22.591Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {40817151-8323-4487-8c1e-fe067729f714},
 private_publication = {false},
 bibtype = {article},
 author = {Nagy, David and Szepesvari, David}
}

@article{
 title = {Analysis of voxel-based 3D object detection methods efficiency for real-time embedded systems},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.10316},
 id = {936cdc1f-7a9b-3c6d-b6d1-bffa2e3897fe},
 created = {2021-07-20T12:45:29.362Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-08T10:20:27.897Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Oleksiienko2021},
 folder_uuids = {cf9189f6-f354-4337-8aaf-a5f12cbf8660,5cd4d7ce-c2fb-4e91-ab80-35deeb123df5},
 private_publication = {false},
 abstract = {Real-time detection of objects in the 3D scene is one of the tasks an autonomous agent needs to perform for understanding its surroundings. While recent Deep Learning-based solutions achieve satisfactory performance, their high computational cost renders their application in real-life settings in which computations need to be performed on embedded platforms intractable. In this paper, we analyze the efficiency of two popular voxel-based 3D object detection methods providing a good compromise between high performance and speed based on two aspects, their ability to detect objects located at large distances from the agent and their ability to operate in real time on embedded platforms equipped with high-performance GPUs. Our experiments show that these methods mostly fail to detect distant small objects due to the sparsity of the input point clouds at large distances. Moreover, models trained on near objects achieve similar or better performance compared to those trained on all objects in the scene. This means that the models learn object appearance representations mostly from near objects. Our findings suggest that a considerable part of the computations of existing methods is focused on locations of the scene that do not contribute with successful detection. This means that the methods can achieve a speed-up of $40$-$60\%$ by restricting operation to near objects while not sacrificing much in performance.},
 bibtype = {article},
 author = {Oleksiienko, Illia and Iosifidis, Alexandros}
}

Real-time detection of objects in the 3D scene is one of the tasks an autonomous agent needs to perform for understanding its surroundings. While recent Deep Learning-based solutions achieve satisfactory performance, their high computational cost renders their application in real-life settings in which computations need to be performed on embedded platforms intractable. In this paper, we analyze the efficiency of two popular voxel-based 3D object detection methods providing a good compromise between high performance and speed based on two aspects, their ability to detect objects located at large distances from the agent and their ability to operate in real time on embedded platforms equipped with high-performance GPUs. Our experiments show that these methods mostly fail to detect distant small objects due to the sparsity of the input point clouds at large distances. Moreover, models trained on near objects achieve similar or better performance compared to those trained on all objects in the scene. This means that the models learn object appearance representations mostly from near objects. Our findings suggest that a considerable part of the computations of existing methods is focused on locations of the scene that do not contribute with successful detection. This means that the methods can achieve a speed-up of $40$-$60\%$ by restricting operation to near objects while not sacrificing much in performance.

@article{
 title = {SIMPLE SPECTRAL GRAPH CONVOLUTION},
 type = {article},
 year = {2021},
 pages = {1-15},
 id = {77317523-de9e-3f5e-8a7f-d79368b3ba82},
 created = {2021-08-04T09:51:19.985Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-08-04T13:05:08.471Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Zhu, Hao and Koniusz, Piotr}
}

@article{
 title = {A Comprehensive Survey on Graph Neural Networks},
 type = {article},
 year = {2021},
 keywords = {Deep learning,graph autoencoder (GAE),graph convolutional networks (GCNs),graph neural networks (GNNs),graph representation learning,network embedding},
 pages = {4-24},
 volume = {32},
 publisher = {IEEE},
 id = {9bdff10a-f897-3446-8652-8a28b7f6b8c5},
 created = {2021-08-17T08:06:02.968Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-30T15:38:16.106Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wu2021},
 folder_uuids = {dbd9a6d6-88f6-4a62-9acd-402fb473145a},
 private_publication = {false},
 abstract = {Deep learning has revolutionized many machine learning tasks in recent years, ranging from image classification and video processing to speech recognition and natural language understanding. The data in these tasks are typically represented in the Euclidean space. However, there is an increasing number of applications, where data are generated from non-Euclidean domains and are represented as graphs with complex relationships and interdependency between objects. The complexity of graph data has imposed significant challenges on the existing machine learning algorithms. Recently, many studies on extending deep learning approaches for graph data have emerged. In this article, we provide a comprehensive overview of graph neural networks (GNNs) in data mining and machine learning fields. We propose a new taxonomy to divide the state-of-The-Art GNNs into four categories, namely, recurrent GNNs, convolutional GNNs, graph autoencoders, and spatial-Temporal GNNs. We further discuss the applications of GNNs across various domains and summarize the open-source codes, benchmark data sets, and model evaluation of GNNs. Finally, we propose potential research directions in this rapidly growing field.},
 bibtype = {article},
 author = {Wu, Zonghan and Pan, Shirui and Chen, Fengwen and Long, Guodong and Zhang, Chengqi and Yu, Philip S.},
 doi = {10.1109/TNNLS.2020.2978386},
 journal = {IEEE Transactions on Neural Networks and Learning Systems},
 number = {1}
}

Deep learning has revolutionized many machine learning tasks in recent years, ranging from image classification and video processing to speech recognition and natural language understanding. The data in these tasks are typically represented in the Euclidean space. However, there is an increasing number of applications, where data are generated from non-Euclidean domains and are represented as graphs with complex relationships and interdependency between objects. The complexity of graph data has imposed significant challenges on the existing machine learning algorithms. Recently, many studies on extending deep learning approaches for graph data have emerged. In this article, we provide a comprehensive overview of graph neural networks (GNNs) in data mining and machine learning fields. We propose a new taxonomy to divide the state-of-The-Art GNNs into four categories, namely, recurrent GNNs, convolutional GNNs, graph autoencoders, and spatial-Temporal GNNs. We further discuss the applications of GNNs across various domains and summarize the open-source codes, benchmark data sets, and model evaluation of GNNs. Finally, we propose potential research directions in this rapidly growing field.

@article{
 title = {Dynamic Convolution for 3D Point Cloud Instance Segmentation},
 type = {article},
 year = {2021},
 keywords = {Index Terms-Point cloud,dynamic convolution,instance segmentation},
 websites = {https://arxiv.org/abs/2107.08392v1},
 month = {7},
 day = {18},
 id = {cf16b23f-c3ca-3dcc-9cb9-abc6640af6b7},
 created = {2021-08-24T10:26:44.691Z},
 accessed = {2021-08-24},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-08-24T10:26:50.322Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {70eb910f-9399-46d8-a4d0-ade5435237b7},
 private_publication = {false},
 abstract = {We propose an approach to instance segmentation from 3D point clouds based on
dynamic convolution. This enables it to adapt, at inference, to varying feature
and object scales. Doing so avoids some pitfalls of bottom up approaches,
including a dependence on hyper-parameter tuning and heuristic post-processing
pipelines to compensate for the inevitable variability in object sizes, even
within a single scene. The representation capability of the network is greatly
improved by gathering homogeneous points that have identical semantic
categories and close votes for the geometric centroids. Instances are then
decoded via several simple convolution layers, where the parameters are
generated conditioned on the input. The proposed approach is proposal-free, and
instead exploits a convolution process that adapts to the spatial and semantic
characteristics of each instance. A light-weight transformer, built on the
bottleneck layer, allows the model to capture long-range dependencies, with
limited computational overhead. The result is a simple, efficient, and robust
approach that yields strong performance on various datasets: ScanNetV2, S3DIS,
and PartNet. The consistent improvements on both voxel- and point-based
architectures imply the effectiveness of the proposed method. Code is available
at: https://git.io/DyCo3D},
 bibtype = {article},
 author = {He, Tong and Shen, Chunhua and Hengel, Anton van den}
}

We propose an approach to instance segmentation from 3D point clouds based on dynamic convolution. This enables it to adapt, at inference, to varying feature and object scales. Doing so avoids some pitfalls of bottom up approaches, including a dependence on hyper-parameter tuning and heuristic post-processing pipelines to compensate for the inevitable variability in object sizes, even within a single scene. The representation capability of the network is greatly improved by gathering homogeneous points that have identical semantic categories and close votes for the geometric centroids. Instances are then decoded via several simple convolution layers, where the parameters are generated conditioned on the input. The proposed approach is proposal-free, and instead exploits a convolution process that adapts to the spatial and semantic characteristics of each instance. A light-weight transformer, built on the bottleneck layer, allows the model to capture long-range dependencies, with limited computational overhead. The result is a simple, efficient, and robust approach that yields strong performance on various datasets: ScanNetV2, S3DIS, and PartNet. The consistent improvements on both voxel- and point-based architectures imply the effectiveness of the proposed method. Code is available at: https://git.io/DyCo3D

@article{
 title = {Tutorial on Variational Autoencoders},
 type = {article},
 year = {2021},
 keywords = {neural networks,prediction,structured,unsupervised learning,variational autoencoders},
 pages = {1-23},
 id = {dfe81fad-91fb-374c-80c1-e7f650b81f9a},
 created = {2021-08-30T18:48:39.015Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:09.729Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Mellon2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4},
 private_publication = {false},
 bibtype = {article},
 author = {Mellon, Carnegie and Berkeley, U C}
}

@article{
 title = {AutoFormer: Searching Transformers for Visual Recognition},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2107.00651v1},
 month = {7},
 day = {1},
 id = {7028c756-1ec0-3190-a4a1-ac290f93c62a},
 created = {2021-08-31T10:52:44.069Z},
 accessed = {2021-08-31},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-02T07:26:36.191Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {Recently, pure transformer-based models have shown great potentials for
vision tasks such as image classification and detection. However, the design of
transformer networks is challenging. It has been observed that the depth,
embedding dimension, and number of heads can largely affect the performance of
vision transformers. Previous models configure these dimensions based upon
manual crafting. In this work, we propose a new one-shot architecture search
framework, namely AutoFormer, dedicated to vision transformer search.
AutoFormer entangles the weights of different blocks in the same layers during
supernet training. Benefiting from the strategy, the trained supernet allows
thousands of subnets to be very well-trained. Specifically, the performance of
these subnets with weights inherited from the supernet is comparable to those
retrained from scratch. Besides, the searched models, which we refer to
AutoFormers, surpass the recent state-of-the-arts such as ViT and DeiT. In
particular, AutoFormer-tiny/small/base achieve 74.7%/81.7%/82.4% top-1 accuracy
on ImageNet with 5.7M/22.9M/53.7M parameters, respectively. Lastly, we verify
the transferability of AutoFormer by providing the performance on downstream
benchmarks and distillation experiments. Code and models are available at
https://github.com/microsoft/AutoML.},
 bibtype = {article},
 author = {Chen, Minghao and Peng, Houwen and Fu, Jianlong and Ling, Haibin}
}

Recently, pure transformer-based models have shown great potentials for vision tasks such as image classification and detection. However, the design of transformer networks is challenging. It has been observed that the depth, embedding dimension, and number of heads can largely affect the performance of vision transformers. Previous models configure these dimensions based upon manual crafting. In this work, we propose a new one-shot architecture search framework, namely AutoFormer, dedicated to vision transformer search. AutoFormer entangles the weights of different blocks in the same layers during supernet training. Benefiting from the strategy, the trained supernet allows thousands of subnets to be very well-trained. Specifically, the performance of these subnets with weights inherited from the supernet is comparable to those retrained from scratch. Besides, the searched models, which we refer to AutoFormers, surpass the recent state-of-the-arts such as ViT and DeiT. In particular, AutoFormer-tiny/small/base achieve 74.7%/81.7%/82.4% top-1 accuracy on ImageNet with 5.7M/22.9M/53.7M parameters, respectively. Lastly, we verify the transferability of AutoFormer by providing the performance on downstream benchmarks and distillation experiments. Code and models are available at https://github.com/microsoft/AutoML.

@article{
 title = {Improving 3D Object Detection with Channel-wise Transformer},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2108.10723v1},
 month = {8},
 day = {23},
 id = {f3e7bbf6-a64e-3d7e-b0ce-45a42c7e4f16},
 created = {2021-08-31T11:11:24.759Z},
 accessed = {2021-08-31},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.050Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Though 3D object detection from point clouds has achieved rapid progress in
recent years, the lack of flexible and high-performance proposal refinement
remains a great hurdle for existing state-of-the-art two-stage detectors.
Previous works on refining 3D proposals have relied on human-designed
components such as keypoints sampling, set abstraction and multi-scale feature
fusion to produce powerful 3D object representations. Such methods, however,
have limited ability to capture rich contextual dependencies among points. In
this paper, we leverage the high-quality region proposal network and a
Channel-wise Transformer architecture to constitute our two-stage 3D object
detection framework (CT3D) with minimal hand-crafted design. The proposed CT3D
simultaneously performs proposal-aware embedding and channel-wise context
aggregation for the point features within each proposal. Specifically, CT3D
uses proposal's keypoints for spatial contextual modelling and learns attention
propagation in the encoding module, mapping the proposal to point embeddings.
Next, a new channel-wise decoding module enriches the query-key interaction via
channel-wise re-weighting to effectively merge multi-level contexts, which
contributes to more accurate object predictions. Extensive experiments
demonstrate that our CT3D method has superior performance and excellent
scalability. Remarkably, CT3D achieves the AP of 81.77% in the moderate car
category on the KITTI test 3D detection benchmark, outperforms state-of-the-art
3D detectors.},
 bibtype = {article},
 author = {Sheng, Hualian and Cai, Sijia and Liu, Yuan and Deng, Bing and Huang, Jianqiang and Hua, Xian-Sheng and Zhao, Min-Jian}
}

Though 3D object detection from point clouds has achieved rapid progress in recent years, the lack of flexible and high-performance proposal refinement remains a great hurdle for existing state-of-the-art two-stage detectors. Previous works on refining 3D proposals have relied on human-designed components such as keypoints sampling, set abstraction and multi-scale feature fusion to produce powerful 3D object representations. Such methods, however, have limited ability to capture rich contextual dependencies among points. In this paper, we leverage the high-quality region proposal network and a Channel-wise Transformer architecture to constitute our two-stage 3D object detection framework (CT3D) with minimal hand-crafted design. The proposed CT3D simultaneously performs proposal-aware embedding and channel-wise context aggregation for the point features within each proposal. Specifically, CT3D uses proposal's keypoints for spatial contextual modelling and learns attention propagation in the encoding module, mapping the proposal to point embeddings. Next, a new channel-wise decoding module enriches the query-key interaction via channel-wise re-weighting to effectively merge multi-level contexts, which contributes to more accurate object predictions. Extensive experiments demonstrate that our CT3D method has superior performance and excellent scalability. Remarkably, CT3D achieves the AP of 81.77% in the moderate car category on the KITTI test 3D detection benchmark, outperforms state-of-the-art 3D detectors.

@article{
 title = {Bottleneck Transformers for Visual Recognition},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2101.11605v2},
 month = {1},
 day = {27},
 id = {95eb8f27-2c19-34e3-8a5b-9462be413192},
 created = {2021-09-01T07:34:53.797Z},
 accessed = {2021-09-01},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-01T07:34:56.861Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {We present BoTNet, a conceptually simple yet powerful backbone architecture
that incorporates self-attention for multiple computer vision tasks including
image classification, object detection and instance segmentation. By just
replacing the spatial convolutions with global self-attention in the final
three bottleneck blocks of a ResNet and no other changes, our approach improves
upon the baselines significantly on instance segmentation and object detection
while also reducing the parameters, with minimal overhead in latency. Through
the design of BoTNet, we also point out how ResNet bottleneck blocks with
self-attention can be viewed as Transformer blocks. Without any bells and
whistles, BoTNet achieves 44.4% Mask AP and 49.7% Box AP on the COCO Instance
Segmentation benchmark using the Mask R-CNN framework; surpassing the previous
best published single model and single scale results of ResNeSt evaluated on
the COCO validation set. Finally, we present a simple adaptation of the BoTNet
design for image classification, resulting in models that achieve a strong
performance of 84.7% top-1 accuracy on the ImageNet benchmark while being up to
1.64x faster in compute time than the popular EfficientNet models on TPU-v3
hardware. We hope our simple and effective approach will serve as a strong
baseline for future research in self-attention models for vision},
 bibtype = {article},
 author = {Srinivas, Aravind and Lin, Tsung-Yi and Parmar, Niki and Shlens, Jonathon and Abbeel, Pieter and Vaswani, Ashish}
}

We present BoTNet, a conceptually simple yet powerful backbone architecture that incorporates self-attention for multiple computer vision tasks including image classification, object detection and instance segmentation. By just replacing the spatial convolutions with global self-attention in the final three bottleneck blocks of a ResNet and no other changes, our approach improves upon the baselines significantly on instance segmentation and object detection while also reducing the parameters, with minimal overhead in latency. Through the design of BoTNet, we also point out how ResNet bottleneck blocks with self-attention can be viewed as Transformer blocks. Without any bells and whistles, BoTNet achieves 44.4% Mask AP and 49.7% Box AP on the COCO Instance Segmentation benchmark using the Mask R-CNN framework; surpassing the previous best published single model and single scale results of ResNeSt evaluated on the COCO validation set. Finally, we present a simple adaptation of the BoTNet design for image classification, resulting in models that achieve a strong performance of 84.7% top-1 accuracy on the ImageNet benchmark while being up to 1.64x faster in compute time than the popular EfficientNet models on TPU-v3 hardware. We hope our simple and effective approach will serve as a strong baseline for future research in self-attention models for vision

@article{
 title = {An Attention Free Transformer},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2105.14103v1},
 month = {5},
 day = {28},
 id = {a915cdb5-6e66-39e4-9ad0-0daeb2ad58d7},
 created = {2021-09-01T08:03:58.653Z},
 accessed = {2021-09-01},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-03T07:05:08.596Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {We introduce Attention Free Transformer (AFT), an efficient variant of
Transformers that eliminates the need for dot product self attention. In an AFT
layer, the key and value are first combined with a set of learned position
biases, the result of which is multiplied with the query in an element-wise
fashion. This new operation has a memory complexity linear w.r.t. both the
context size and the dimension of features, making it compatible to both large
input and model sizes. We also introduce AFT-local and AFT-conv, two model
variants that take advantage of the idea of locality and spatial weight sharing
while maintaining global connectivity. We conduct extensive experiments on two
autoregressive modeling tasks (CIFAR10 and Enwik8) as well as an image
recognition task (ImageNet-1K classification). We show that AFT demonstrates
competitive performance on all the benchmarks, while providing excellent
efficiency at the same time.},
 bibtype = {article},
 author = {Zhai, Shuangfei and Talbott, Walter and Srivastava, Nitish and Huang, Chen and Goh, Hanlin and Zhang, Ruixiang and Susskind, Josh}
}

We introduce Attention Free Transformer (AFT), an efficient variant of Transformers that eliminates the need for dot product self attention. In an AFT layer, the key and value are first combined with a set of learned position biases, the result of which is multiplied with the query in an element-wise fashion. This new operation has a memory complexity linear w.r.t. both the context size and the dimension of features, making it compatible to both large input and model sizes. We also introduce AFT-local and AFT-conv, two model variants that take advantage of the idea of locality and spatial weight sharing while maintaining global connectivity. We conduct extensive experiments on two autoregressive modeling tasks (CIFAR10 and Enwik8) as well as an image recognition task (ImageNet-1K classification). We show that AFT demonstrates competitive performance on all the benchmarks, while providing excellent efficiency at the same time.

@article{
 title = {Tokens-to-Token ViT: Training Vision Transformers from Scratch on ImageNet},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2101.11986v1},
 month = {1},
 day = {28},
 id = {b1b7328f-2ca1-3670-97ce-8142b0a5d0a8},
 created = {2021-09-01T08:06:14.613Z},
 accessed = {2021-09-01},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-01T08:06:23.156Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {Transformers, which are popular for language modeling, have been explored for
solving vision tasks recently, e.g., the Vision Transformers (ViT) for image
classification. The ViT model splits each image into a sequence of tokens with
fixed length and then applies multiple Transformer layers to model their global
relation for classification. However, ViT achieves inferior performance
compared with CNNs when trained from scratch on a midsize dataset (e.g.,
ImageNet). We find it is because: 1) the simple tokenization of input images
fails to model the important local structure (e.g., edges, lines) among
neighboring pixels, leading to its low training sample efficiency; 2) the
redundant attention backbone design of ViT leads to limited feature richness in
fixed computation budgets and limited training samples. To overcome such limitations, we propose a new Tokens-To-Token Vision
Transformers (T2T-ViT), which introduces 1) a layer-wise Tokens-to-Token (T2T)
transformation to progressively structurize the image to tokens by recursively
aggregating neighboring Tokens into one Token (Tokens-to-Token), such that
local structure presented by surrounding tokens can be modeled and tokens
length can be reduced; 2) an efficient backbone with a deep-narrow structure
for vision transformers motivated by CNN architecture design after extensive
study. Notably, T2T-ViT reduces the parameter counts and MACs of vanilla ViT by
200\%, while achieving more than 2.5\% improvement when trained from scratch on
ImageNet. It also outperforms ResNets and achieves comparable performance with
MobileNets when directly training on ImageNet. For example, T2T-ViT with
ResNet50 comparable size can achieve 80.7\% top-1 accuracy on ImageNet. (Code:
https://github.com/yitu-opensource/T2T-ViT)},
 bibtype = {article},
 author = {Yuan, Li and Chen, Yunpeng and Wang, Tao and Yu, Weihao and Shi, Yujun and Tay, Francis EH and Feng, Jiashi and Yan, Shuicheng}
}

Transformers, which are popular for language modeling, have been explored for solving vision tasks recently, e.g., the Vision Transformers (ViT) for image classification. The ViT model splits each image into a sequence of tokens with fixed length and then applies multiple Transformer layers to model their global relation for classification. However, ViT achieves inferior performance compared with CNNs when trained from scratch on a midsize dataset (e.g., ImageNet). We find it is because: 1) the simple tokenization of input images fails to model the important local structure (e.g., edges, lines) among neighboring pixels, leading to its low training sample efficiency; 2) the redundant attention backbone design of ViT leads to limited feature richness in fixed computation budgets and limited training samples. To overcome such limitations, we propose a new Tokens-To-Token Vision Transformers (T2T-ViT), which introduces 1) a layer-wise Tokens-to-Token (T2T) transformation to progressively structurize the image to tokens by recursively aggregating neighboring Tokens into one Token (Tokens-to-Token), such that local structure presented by surrounding tokens can be modeled and tokens length can be reduced; 2) an efficient backbone with a deep-narrow structure for vision transformers motivated by CNN architecture design after extensive study. Notably, T2T-ViT reduces the parameter counts and MACs of vanilla ViT by 200\%, while achieving more than 2.5\% improvement when trained from scratch on ImageNet. It also outperforms ResNets and achieves comparable performance with MobileNets when directly training on ImageNet. For example, T2T-ViT with ResNet50 comparable size can achieve 80.7\% top-1 accuracy on ImageNet. (Code: https://github.com/yitu-opensource/T2T-ViT)

@article{
 title = {Recent Advances in Variational Autoencoders with Representation Learning for Biomedical Informatics: A Survey},
 type = {article},
 year = {2021},
 keywords = {Deep learning,biomedical informatics,data representation,generative models,latent space,representation learning,unsupervised learning,variational autoencoders (VAEs)},
 pages = {4939-4956},
 volume = {9},
 id = {3d37dbea-cc6c-3b76-9071-c8fcf066f56c},
 created = {2021-09-02T05:25:53.249Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:07.833Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wei2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4},
 private_publication = {false},
 abstract = {Variational autoencoders (VAEs) are deep latent space generative models that have been immensely successful in multiple exciting applications in biomedical informatics such as molecular design, protein design, medical image classification and segmentation, integrated multi-omics data analyses, and large-scale biological sequence analyses, among others. The fundamental idea in VAEs is to learn the distribution of data in such a way that new meaningful data with more intra-class variations can be generated from the encoded distribution. The ability of VAEs to synthesize new data with more representation variance at state-of-art levels provides hope that the chronic scarcity of labeled data in the biomedical field can be resolved. Furthermore, VAEs have made nonlinear latent variable models tractable for modeling complex distributions. This has allowed for efficient extraction of relevant biomedical information from learned features for biological data sets, referred to as unsupervised feature representation learning. In this article, we review the various recent advancements in the development and application of VAEs for biomedical informatics. We discuss challenges and future opportunities for biomedical research with respect to VAEs.},
 bibtype = {article},
 author = {Wei, Ruoqi and Mahmood, Ausif},
 doi = {10.1109/ACCESS.2020.3048309},
 journal = {IEEE Access}
}

Variational autoencoders (VAEs) are deep latent space generative models that have been immensely successful in multiple exciting applications in biomedical informatics such as molecular design, protein design, medical image classification and segmentation, integrated multi-omics data analyses, and large-scale biological sequence analyses, among others. The fundamental idea in VAEs is to learn the distribution of data in such a way that new meaningful data with more intra-class variations can be generated from the encoded distribution. The ability of VAEs to synthesize new data with more representation variance at state-of-art levels provides hope that the chronic scarcity of labeled data in the biomedical field can be resolved. Furthermore, VAEs have made nonlinear latent variable models tractable for modeling complex distributions. This has allowed for efficient extraction of relevant biomedical information from learned features for biological data sets, referred to as unsupervised feature representation learning. In this article, we review the various recent advancements in the development and application of VAEs for biomedical informatics. We discuss challenges and future opportunities for biomedical research with respect to VAEs.

@article{
 title = {A comprehensive study of autoencoders' applications related to images},
 type = {article},
 year = {2021},
 keywords = {Autoencoders,CNN,DNN,Generative models,VAE},
 pages = {43-54},
 volume = {2845},
 id = {e5e7890f-095c-3d0f-b21c-4f32a51e3474},
 created = {2021-09-02T05:25:53.358Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:07.860Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Kovenko2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,8ab28090-4ad8-4e96-920f-bd4344219976},
 private_publication = {false},
 abstract = {This article incorporates a comprehensive study of autoencoders' applications related to images. First of all, a vanilla autoencoder is described along with details of its architecture and training procedure. Secondly, main methods for regularization of it are exposed, such as dropout and additive gaussian noise. The applications of autoencoders such as image morphing, reconstruction and search are shown. Then, the VAE (variational autoencoder) is highlighted. Main applications of it such as outliers detection and image generation are described. Finally, it's shown that using warm-up for VAE with respect to KL loss gives much more plausible results in terms of image generation.},
 bibtype = {article},
 author = {Kovenko, Volodymyr and Bogach, Ilona},
 journal = {CEUR Workshop Proceedings}
}

This article incorporates a comprehensive study of autoencoders' applications related to images. First of all, a vanilla autoencoder is described along with details of its architecture and training procedure. Secondly, main methods for regularization of it are exposed, such as dropout and additive gaussian noise. The applications of autoencoders such as image morphing, reconstruction and search are shown. Then, the VAE (variational autoencoder) is highlighted. Main applications of it such as outliers detection and image generation are described. Finally, it's shown that using warm-up for VAE with respect to KL loss gives much more plausible results in terms of image generation.

@article{
 title = {A Survey on Variational Autoencoders from a Green AI Perspective},
 type = {article},
 year = {2021},
 keywords = {generative modeling,greenai,variational autoencoders},
 volume = {2},
 id = {4a000622-f6e4-3741-a3a0-14761a21e3e9},
 created = {2021-09-02T05:25:53.376Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-17T07:10:38.620Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Asperti2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,8ab28090-4ad8-4e96-920f-bd4344219976},
 private_publication = {false},
 abstract = {Variational Autoencoders (VAEs) are powerful generative models that merge elements from statistics and information theory with the flexibility offered by deep neural networks to efficiently solve the generation problem for high-dimensional data. The key insight of VAEs is to learn the latent distribution of data in such a way that new meaningful samples can be generated from it. This approach led to tremendous research and variations in the architectural design of VAEs, nourishing the recent field of research known as unsupervised representation learning. In this article, we provide a comparative evaluation of some of the most successful, recent variations of VAEs. We particularly focus the analysis on the energetic efficiency of the different models, in the spirit of the so-called Green AI, aiming both to reduce the carbon footprint and the financial cost of generative techniques. For each architecture, we provide its mathematical formulation, the ideas underlying its design, a detailed model description, a running implementation and quantitative results.},
 bibtype = {article},
 author = {Asperti, Andrea and Evangelista, Davide and Loli Piccolomini, Elena},
 doi = {10.1007/s42979-021-00702-9},
 journal = {SN Computer Science},
 number = {4}
}

Variational Autoencoders (VAEs) are powerful generative models that merge elements from statistics and information theory with the flexibility offered by deep neural networks to efficiently solve the generation problem for high-dimensional data. The key insight of VAEs is to learn the latent distribution of data in such a way that new meaningful samples can be generated from it. This approach led to tremendous research and variations in the architectural design of VAEs, nourishing the recent field of research known as unsupervised representation learning. In this article, we provide a comparative evaluation of some of the most successful, recent variations of VAEs. We particularly focus the analysis on the energetic efficiency of the different models, in the spirit of the so-called Green AI, aiming both to reduce the carbon footprint and the financial cost of generative techniques. For each architecture, we provide its mathematical formulation, the ideas underlying its design, a detailed model description, a running implementation and quantitative results.

@article{
 title = {Random Feature Attention},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.02143},
 month = {3},
 day = {3},
 id = {982c487a-79c0-3f2e-8c9b-03e280cf46e6},
 created = {2021-09-03T07:05:48.453Z},
 accessed = {2021-09-03},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-03T07:05:51.494Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {Transformers are state-of-the-art models for a variety of sequence modeling tasks. At their core is an attention function which models pairwise interactions between the inputs at every timestep. While attention is powerful, it does not scale efficiently to long sequences due to its quadratic time and space complexity in the sequence length. We propose RFA, a linear time and space attention that uses random feature methods to approximate the softmax function, and explore its application in transformers. RFA can be used as a drop-in replacement for conventional softmax attention and offers a straightforward way of learning with recency bias through an optional gating mechanism. Experiments on language modeling and machine translation demonstrate that RFA achieves similar or better performance compared to strong transformer baselines. In the machine translation experiment, RFA decodes twice as fast as a vanilla transformer. Compared to existing efficient transformer variants, RFA is competitive in terms of both accuracy and efficiency on three long text classification datasets. Our analysis shows that RFA's efficiency gains are especially notable on long sequences, suggesting that RFA will be particularly useful in tasks that require working with large inputs, fast decoding speed, or low memory footprints.},
 bibtype = {article},
 author = {Peng, Hao and Pappas, Nikolaos and Yogatama, Dani and Schwartz, Roy and Smith, Noah A. and Kong, Lingpeng}
}

Transformers are state-of-the-art models for a variety of sequence modeling tasks. At their core is an attention function which models pairwise interactions between the inputs at every timestep. While attention is powerful, it does not scale efficiently to long sequences due to its quadratic time and space complexity in the sequence length. We propose RFA, a linear time and space attention that uses random feature methods to approximate the softmax function, and explore its application in transformers. RFA can be used as a drop-in replacement for conventional softmax attention and offers a straightforward way of learning with recency bias through an optional gating mechanism. Experiments on language modeling and machine translation demonstrate that RFA achieves similar or better performance compared to strong transformer baselines. In the machine translation experiment, RFA decodes twice as fast as a vanilla transformer. Compared to existing efficient transformer variants, RFA is competitive in terms of both accuracy and efficiency on three long text classification datasets. Our analysis shows that RFA's efficiency gains are especially notable on long sequences, suggesting that RFA will be particularly useful in tasks that require working with large inputs, fast decoding speed, or low memory footprints.

@article{
 title = {LambdaNetworks: Modeling Long-Range Interactions Without Attention},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2102.08602v1},
 month = {2},
 day = {17},
 id = {19fb7926-8c5e-3c03-8917-a2ed605c820a},
 created = {2021-09-03T07:06:05.730Z},
 accessed = {2021-09-03},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-03T07:06:08.788Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8d050117-e419-4b32-ad70-c875c74fa2b4},
 private_publication = {false},
 abstract = {We present lambda layers -- an alternative framework to self-attention -- for
capturing long-range interactions between an input and structured contextual
information (e.g. a pixel surrounded by other pixels). Lambda layers capture
such interactions by transforming available contexts into linear functions,
termed lambdas, and applying these linear functions to each input separately.
Similar to linear attention, lambda layers bypass expensive attention maps, but
in contrast, they model both content and position-based interactions which
enables their application to large structured inputs such as images. The
resulting neural network architectures, LambdaNetworks, significantly
outperform their convolutional and attentional counterparts on ImageNet
classification, COCO object detection and COCO instance segmentation, while
being more computationally efficient. Additionally, we design LambdaResNets, a
family of hybrid architectures across different scales, that considerably
improves the speed-accuracy tradeoff of image classification models.
LambdaResNets reach excellent accuracies on ImageNet while being 3.2 - 4.4x
faster than the popular EfficientNets on modern machine learning accelerators.
When training with an additional 130M pseudo-labeled images, LambdaResNets
achieve up to a 9.5x speed-up over the corresponding EfficientNet checkpoints.},
 bibtype = {article},
 author = {Bello, Irwan}
}

We present lambda layers -- an alternative framework to self-attention -- for capturing long-range interactions between an input and structured contextual information (e.g. a pixel surrounded by other pixels). Lambda layers capture such interactions by transforming available contexts into linear functions, termed lambdas, and applying these linear functions to each input separately. Similar to linear attention, lambda layers bypass expensive attention maps, but in contrast, they model both content and position-based interactions which enables their application to large structured inputs such as images. The resulting neural network architectures, LambdaNetworks, significantly outperform their convolutional and attentional counterparts on ImageNet classification, COCO object detection and COCO instance segmentation, while being more computationally efficient. Additionally, we design LambdaResNets, a family of hybrid architectures across different scales, that considerably improves the speed-accuracy tradeoff of image classification models. LambdaResNets reach excellent accuracies on ImageNet while being 3.2 - 4.4x faster than the popular EfficientNets on modern machine learning accelerators. When training with an additional 130M pseudo-labeled images, LambdaResNets achieve up to a 9.5x speed-up over the corresponding EfficientNet checkpoints.

@article{
 title = {Cross-Modal Center Loss for 3D Cross-Modal Retrieval},
 type = {article},
 year = {2021},
 pages = {3142-3151},
 id = {4f31dd5a-26eb-32eb-aa9c-9696d494d166},
 created = {2021-09-07T08:57:34.490Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-08T09:21:50.970Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {a6fefa10-ad39-4ee5-850c-dcbd4fed6307},
 private_publication = {false},
 abstract = {Cross-modal retrieval aims to learn discriminative and modal-invariant features for data from different modalities. Unlike the existing methods which usually learn from the features extracted by offline networks, in this paper, we propose an approach to jointly train the components of cross-modal retrieval framework with metadata, and enable the network to find optimal features. The proposed end-to-end framework is updated with three loss functions: 1) a novel cross-modal center loss to eliminate cross-modal discrepancy , 2) cross-entropy loss to maximize inter-class variations , and 3) mean-square-error loss to reduce modality variations. In particular, our proposed cross-modal center loss minimizes the distances of features from objects belonging to the same class across all modalities. Extensive experiments have been conducted on the retrieval tasks across multi-modalities including 2D image, 3D point cloud and mesh data. The proposed framework significantly out-performs the state-of-the-art methods for both cross-modal and in-domain retrieval for 3D objects on the ModelNet10 and ModelNet40 datasets.},
 bibtype = {article},
 author = {Jing, Longlong and Vahdani, Elahe and Tan, Jiaxing and Tian, Yingli}
}

Cross-modal retrieval aims to learn discriminative and modal-invariant features for data from different modalities. Unlike the existing methods which usually learn from the features extracted by offline networks, in this paper, we propose an approach to jointly train the components of cross-modal retrieval framework with metadata, and enable the network to find optimal features. The proposed end-to-end framework is updated with three loss functions: 1) a novel cross-modal center loss to eliminate cross-modal discrepancy , 2) cross-entropy loss to maximize inter-class variations , and 3) mean-square-error loss to reduce modality variations. In particular, our proposed cross-modal center loss minimizes the distances of features from objects belonging to the same class across all modalities. Extensive experiments have been conducted on the retrieval tasks across multi-modalities including 2D image, 3D point cloud and mesh data. The proposed framework significantly out-performs the state-of-the-art methods for both cross-modal and in-domain retrieval for 3D objects on the ModelNet10 and ModelNet40 datasets.

@article{
 title = {Toward Unsupervised 3d Point Cloud Anomaly Detection Using Variational Autoencoder},
 type = {article},
 year = {2021},
 pages = {3118-3122},
 publisher = {IEEE},
 id = {b5a0b82b-b3d7-3fc6-80ff-dc98f72e0e00},
 created = {2021-09-09T14:35:21.257Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:09.861Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Masuda2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 bibtype = {article},
 author = {Masuda, Mana and Hachiuma, Ryo and Fujii, Ryo and Saito, Hideo and Sekikawa, Yusuke},
 doi = {10.1109/icip42928.2021.9506795}
}

@article{
 title = {Advances in agriculture robotics: A state-of-the-art review and challenges ahead},
 type = {article},
 year = {2021},
 keywords = {Agricultural robots,Agriculture 4.0,Precision agriculture},
 pages = {1-31},
 volume = {10},
 id = {bc1196de-20a2-3dfb-8f7f-3e6d4c3e4636},
 created = {2021-09-14T08:01:18.690Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-14T08:42:57.265Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Oliveira2021},
 private_publication = {false},
 abstract = {The constant advances in agricultural robotics aim to overcome the challenges imposed by population growth, accelerated urbanization, high competitiveness of high-quality products, environmental preservation and a lack of qualified labor. In this sense, this review paper surveys the main existing applications of agricultural robotic systems for the execution of land preparation before planting, sowing, planting, plant treatment, harvesting, yield estimation and phenotyping. In general, all robots were evaluated according to the following criteria: its locomotion system, what is the final application, if it has sensors, robotic arm and/or computer vision algorithm, what is its development stage and which country and continent they belong. After evaluating all similar characteristics, to expose the research trends, common pitfalls and the characteristics that hinder commercial development, and discover which countries are investing into Research and Development (R&D) in these technologies for the future, four major areas that need future research work for enhancing the state of the art in smart agriculture were highlighted: locomotion systems, sensors, computer vision algorithms and communication technologies. The results of this research suggest that the investment in agricultural robotic systems allows to achieve short—harvest monitoring—and long-term objectives—yield estimation.},
 bibtype = {article},
 author = {Oliveira, Luiz F.P. and Moreira, António P. and Silva, Manuel F.},
 doi = {10.3390/robotics10020052},
 journal = {Robotics},
 number = {2}
}

The constant advances in agricultural robotics aim to overcome the challenges imposed by population growth, accelerated urbanization, high competitiveness of high-quality products, environmental preservation and a lack of qualified labor. In this sense, this review paper surveys the main existing applications of agricultural robotic systems for the execution of land preparation before planting, sowing, planting, plant treatment, harvesting, yield estimation and phenotyping. In general, all robots were evaluated according to the following criteria: its locomotion system, what is the final application, if it has sensors, robotic arm and/or computer vision algorithm, what is its development stage and which country and continent they belong. After evaluating all similar characteristics, to expose the research trends, common pitfalls and the characteristics that hinder commercial development, and discover which countries are investing into Research and Development (R&D) in these technologies for the future, four major areas that need future research work for enhancing the state of the art in smart agriculture were highlighted: locomotion systems, sensors, computer vision algorithms and communication technologies. The results of this research suggest that the investment in agricultural robotic systems allows to achieve short—harvest monitoring—and long-term objectives—yield estimation.

@article{
 title = {Transformers in Vision: A Survey},
 type = {article},
 year = {2021},
 pages = {1-28},
 websites = {http://arxiv.org/abs/2101.01169-----},
 id = {8ea81367-abc1-3f3f-804e-3676eeacc7a5},
 created = {2021-09-24T06:12:48.936Z},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-24T06:12:55.387Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d54ba66b-a8cf-41de-8e2d-c3256f322e07},
 private_publication = {false},
 abstract = {Astounding results from Transformer models on natural language tasks have intrigued the vision community to study their application to computer vision problems. Among their salient benefits, Transformers enable modeling long dependencies between input sequence elements and support parallel processing of sequence as compared to recurrent networks e.g., Long short-term memory (LSTM). Different from convolutional networks, Transformers require minimal inductive biases for their design and are naturally suited as set-functions. Furthermore, the straightforward design of Transformers allows processing multiple modalities (e.g., images, videos, text and speech) using similar processing blocks and demonstrates excellent scalability to very large capacity networks and huge datasets. These strengths have led to exciting progress on a number of vision tasks using Transformer networks. This survey aims to provide a comprehensive overview of the Transformer models in the computer vision discipline. We start with an introduction to fundamental concepts behind the success of Transformers i.e., self-attention, large-scale pre-training, and bidirectional encoding. We then cover extensive applications of transformers in vision including popular recognition tasks (e.g., image classification, object detection, action recognition, and segmentation), generative modeling, multi-modal tasks (e.g., visual-question answering, visual reasoning, and visual grounding), video processing (e.g., activity recognition, video forecasting), low-level vision (e.g., image super-resolution, image enhancement, and colorization) and 3D analysis (e.g., point cloud classification and segmentation). We compare the respective advantages and limitations of popular techniques both in terms of architectural design and their experimental value. Finally, we provide an analysis on open research directions and possible future works.},
 bibtype = {article},
 author = {Khan, Salman and Naseer, Muzammal and Hayat, Munawar and Zamir, Syed Waqas and Khan, Fahad Shahbaz and Shah, Mubarak}
}

Astounding results from Transformer models on natural language tasks have intrigued the vision community to study their application to computer vision problems. Among their salient benefits, Transformers enable modeling long dependencies between input sequence elements and support parallel processing of sequence as compared to recurrent networks e.g., Long short-term memory (LSTM). Different from convolutional networks, Transformers require minimal inductive biases for their design and are naturally suited as set-functions. Furthermore, the straightforward design of Transformers allows processing multiple modalities (e.g., images, videos, text and speech) using similar processing blocks and demonstrates excellent scalability to very large capacity networks and huge datasets. These strengths have led to exciting progress on a number of vision tasks using Transformer networks. This survey aims to provide a comprehensive overview of the Transformer models in the computer vision discipline. We start with an introduction to fundamental concepts behind the success of Transformers i.e., self-attention, large-scale pre-training, and bidirectional encoding. We then cover extensive applications of transformers in vision including popular recognition tasks (e.g., image classification, object detection, action recognition, and segmentation), generative modeling, multi-modal tasks (e.g., visual-question answering, visual reasoning, and visual grounding), video processing (e.g., activity recognition, video forecasting), low-level vision (e.g., image super-resolution, image enhancement, and colorization) and 3D analysis (e.g., point cloud classification and segmentation). We compare the respective advantages and limitations of popular techniques both in terms of architectural design and their experimental value. Finally, we provide an analysis on open research directions and possible future works.

@article{
 title = {Attention Models for Point Clouds in Deep Learning: A Survey},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2102.10788},
 id = {74d676bb-c068-3a48-8a5a-925bc078831d},
 created = {2021-09-27T07:36:18.508Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-17T16:08:03.344Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Recently, the advancement of 3D point clouds in deep learning has attracted intensive research in different application domains such as computer vision and robotic tasks. However, creating feature representation of robust, discriminative from unordered and irregular point clouds is challenging. In this paper, our ultimate goal is to provide a comprehensive overview of the point clouds feature representation which uses attention models. More than 75+ key contributions in the recent three years are summarized in this survey, including the 3D objective detection, 3D semantic segmentation, 3D pose estimation, point clouds completion etc. We provide a detailed characterization (1) the role of attention mechanisms, (2) the usability of attention models into different tasks, (3) the development trend of key technology.},
 bibtype = {article},
 author = {Wang, Xu and Jin, Yi and Cen, Yigang and Wang, Tao and Li, Yidong}
}

Recently, the advancement of 3D point clouds in deep learning has attracted intensive research in different application domains such as computer vision and robotic tasks. However, creating feature representation of robust, discriminative from unordered and irregular point clouds is challenging. In this paper, our ultimate goal is to provide a comprehensive overview of the point clouds feature representation which uses attention models. More than 75+ key contributions in the recent three years are summarized in this survey, including the 3D objective detection, 3D semantic segmentation, 3D pose estimation, point clouds completion etc. We provide a detailed characterization (1) the role of attention mechanisms, (2) the usability of attention models into different tasks, (3) the development trend of key technology.

@article{
 title = {GCN-Denoiser: Mesh Denoising with Graph Convolutional Networks},
 type = {article},
 year = {2021},
 keywords = {Mesh denoising, graph convolutional networks, casc},
 volume = {40},
 websites = {http://arxiv.org/abs/2108.05128},
 publisher = {Association for Computing Machinery},
 id = {23402ab5-4c00-3c47-a4ab-233db1b098da},
 created = {2021-09-27T07:36:18.511Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-17T16:08:03.611Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {In this paper, we present GCN-Denoiser, a novel feature-preserving mesh denoising method based on graph convolutional networks (GCNs). Unlike previous learning-based mesh denoising methods that exploit hand-crafted or voxel-based representations for feature learning, our method explores the structure of a triangular mesh itself and introduces a graph representation followed by graph convolution operations in the dual space of triangles. We show such a graph representation naturally captures the geometry features while being lightweight for both training and inference. To facilitate effective feature learning, our network exploits both static and dynamic edge convolutions, which allow us to learn information from both the explicit mesh structure and potential implicit relations among unconnected neighbors. To better approximate an unknown noise function, we introduce a cascaded optimization paradigm to progressively regress the noise-free facet normals with multiple GCNs. GCN-Denoiser achieves the new state-of-the-art results in multiple noise datasets, including CAD models often containing sharp features and raw scan models with real noise captured from different devices. We also create a new dataset called PrintData containing 20 real scans with their corresponding ground-truth meshes for the research community. Our code and data are available in https://github.com/Jhonve/GCN-Denoiser.},
 bibtype = {article},
 author = {Shen, Yuefan and Fu, Hongbo and Du, Zhongshuo and Chen, Xiang and Burnaev, Evgeny and Zorin, Denis and Zhou, Kun and Zheng, Youyi},
 doi = {10.1145/3480168},
 journal = {ACM Transactions on Graphics},
 number = {4}
}

In this paper, we present GCN-Denoiser, a novel feature-preserving mesh denoising method based on graph convolutional networks (GCNs). Unlike previous learning-based mesh denoising methods that exploit hand-crafted or voxel-based representations for feature learning, our method explores the structure of a triangular mesh itself and introduces a graph representation followed by graph convolution operations in the dual space of triangles. We show such a graph representation naturally captures the geometry features while being lightweight for both training and inference. To facilitate effective feature learning, our network exploits both static and dynamic edge convolutions, which allow us to learn information from both the explicit mesh structure and potential implicit relations among unconnected neighbors. To better approximate an unknown noise function, we introduce a cascaded optimization paradigm to progressively regress the noise-free facet normals with multiple GCNs. GCN-Denoiser achieves the new state-of-the-art results in multiple noise datasets, including CAD models often containing sharp features and raw scan models with real noise captured from different devices. We also create a new dataset called PrintData containing 20 real scans with their corresponding ground-truth meshes for the research community. Our code and data are available in https://github.com/Jhonve/GCN-Denoiser.

@article{
 title = {GAPointNet: Graph attention based point neural network for exploiting local feature of point cloud},
 type = {article},
 year = {2021},
 keywords = {Attention pooling,Graph attention,Multiple heads mechanism,Point cloud,Semantic segmentation,Shape classification},
 pages = {122-132},
 volume = {438},
 id = {7059fc2a-f6cf-37e5-8757-9b9cc7694e23},
 created = {2021-09-27T07:36:18.512Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-09-30T06:29:44.030Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8},
 private_publication = {false},
 abstract = {Exploiting fine-grained semantic features on point cloud data is still challenging because of its irregular and sparse structure in a non-Euclidean space. In order to represent the local feature for each central point that is helpful towards better contextual learning, a max pooling operation is often used to highlight the most important feature in the local region. However, all other geometric local correlations between each central point and corresponding neighbourhood are ignored during the max pooling operation. To this end, the attention mechanism is promising in capturing node representation on graph-based data by attending over all the neighbouring nodes. In this paper, we propose a novel neural network for point cloud analysis, GAPointNet, which is able to learn local geometric representations by embedding graph attention mechanism within stacked Multi-Layer-Perceptron (MLP) layers. Specifically, we highlight different attention weights on the neighbourhood of each center point to efficiently exploit local features. We also combine attention features with local signature features generated by our attention pooling to fully extract local geometric structures and enhance the network robustness. The proposed GAPointNet architecture is tested on various benchmark datasets (i.e. ModelNet40, ShapeNet part, S3DIS, KITTI) and achieves state-of-the-art performance in both the shape classification and segmentation tasks.},
 bibtype = {article},
 author = {Chen, Can and Fragonara, Luca Zanotti and Tsourdos, Antonios},
 doi = {10.1016/j.neucom.2021.01.095},
 journal = {Neurocomputing}
}

Exploiting fine-grained semantic features on point cloud data is still challenging because of its irregular and sparse structure in a non-Euclidean space. In order to represent the local feature for each central point that is helpful towards better contextual learning, a max pooling operation is often used to highlight the most important feature in the local region. However, all other geometric local correlations between each central point and corresponding neighbourhood are ignored during the max pooling operation. To this end, the attention mechanism is promising in capturing node representation on graph-based data by attending over all the neighbouring nodes. In this paper, we propose a novel neural network for point cloud analysis, GAPointNet, which is able to learn local geometric representations by embedding graph attention mechanism within stacked Multi-Layer-Perceptron (MLP) layers. Specifically, we highlight different attention weights on the neighbourhood of each center point to efficiently exploit local features. We also combine attention features with local signature features generated by our attention pooling to fully extract local geometric structures and enhance the network robustness. The proposed GAPointNet architecture is tested on various benchmark datasets (i.e. ModelNet40, ShapeNet part, S3DIS, KITTI) and achieves state-of-the-art performance in both the shape classification and segmentation tasks.

@article{
 title = {Graph Attention Networks for Point Cloud Processing},
 type = {article},
 year = {2021},
 id = {340c2f37-2d07-3d71-9794-ce2988a56200},
 created = {2021-09-27T07:36:18.554Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-17T16:08:04.598Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Thakur, Sumesh and Scotia, Nova and Scotia, Nova and Thakur, Copyright Sumesh and Examiner, External},
 number = {July}
}

@article{
 title = {GAPointNet: Graph attention based point neural network for exploiting local feature of point cloud},
 type = {article},
 year = {2021},
 keywords = {Attention pooling,Graph attention,Multiple heads mechanism,Point cloud,Semantic segmentation,Shape classification},
 pages = {122-132},
 volume = {438},
 id = {e4354bd5-85b1-3218-91c2-41db35603705},
 created = {2021-09-28T11:26:08.757Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-17T16:08:02.335Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Exploiting fine-grained semantic features on point cloud data is still challenging because of its irregular and sparse structure in a non-Euclidean space. In order to represent the local feature for each central point that is helpful towards better contextual learning, a max pooling operation is often used to highlight the most important feature in the local region. However, all other geometric local correlations between each central point and corresponding neighbourhood are ignored during the max pooling operation. To this end, the attention mechanism is promising in capturing node representation on graph-based data by attending over all the neighbouring nodes. In this paper, we propose a novel neural network for point cloud analysis, GAPointNet, which is able to learn local geometric representations by embedding graph attention mechanism within stacked Multi-Layer-Perceptron (MLP) layers. Specifically, we highlight different attention weights on the neighbourhood of each center point to efficiently exploit local features. We also combine attention features with local signature features generated by our attention pooling to fully extract local geometric structures and enhance the network robustness. The proposed GAPointNet architecture is tested on various benchmark datasets (i.e. ModelNet40, ShapeNet part, S3DIS, KITTI) and achieves state-of-the-art performance in both the shape classification and segmentation tasks.},
 bibtype = {article},
 author = {Chen, Can and Fragonara, Luca Zanotti and Tsourdos, Antonios},
 doi = {10.1016/j.neucom.2021.01.095},
 journal = {Neurocomputing}
}

Exploiting fine-grained semantic features on point cloud data is still challenging because of its irregular and sparse structure in a non-Euclidean space. In order to represent the local feature for each central point that is helpful towards better contextual learning, a max pooling operation is often used to highlight the most important feature in the local region. However, all other geometric local correlations between each central point and corresponding neighbourhood are ignored during the max pooling operation. To this end, the attention mechanism is promising in capturing node representation on graph-based data by attending over all the neighbouring nodes. In this paper, we propose a novel neural network for point cloud analysis, GAPointNet, which is able to learn local geometric representations by embedding graph attention mechanism within stacked Multi-Layer-Perceptron (MLP) layers. Specifically, we highlight different attention weights on the neighbourhood of each center point to efficiently exploit local features. We also combine attention features with local signature features generated by our attention pooling to fully extract local geometric structures and enhance the network robustness. The proposed GAPointNet architecture is tested on various benchmark datasets (i.e. ModelNet40, ShapeNet part, S3DIS, KITTI) and achieves state-of-the-art performance in both the shape classification and segmentation tasks.

@article{
 title = {View-Aware Geometry-Structure Joint Learning for Single-View 3D Shape Reconstruction},
 type = {article},
 year = {2021},
 keywords = {Geometry,Image reconstruction,Multimodal Learning,Periodic structures,Representation Learning,Shape,Single-View 3D Reconstruction,Solid modeling,Structure-Aware Reconstruction,Three-dimensional displays,Topology},
 pages = {1-16},
 volume = {8828},
 publisher = {IEEE},
 id = {e02172c6-db18-3f85-9b16-5f49378e0656},
 created = {2021-09-29T10:16:08.708Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:09.052Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Zhang2021},
 folder_uuids = {a6db5ca6-7f95-48a4-bc40-9e41eea78434,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {Reconstructing a 3D shape from a single-view image using deep learning is becoming increasingly popular recently. Most existing methods only focus on reconstructing the 3D shape geometry based on the image constraint. The lack of explicit modeling of structure relations among shape parts yields low-quality reconstruction results for structure-rich man-made shapes. In addition, conventional 2D-3D joint embedding architecture for image-based 3D shape reconstruction often omits the specific view information from the given image, which may lead to degraded geometry and structure reconstruction. We address these problems by introducing VGSNet, an encoder-decoder architecture for view-aware joint geometry and structure learning. The key idea is to jointly learn a multimodal feature representation of 2D image, 3D shape geometry and structure so that both geometry and structure details can be reconstructed from a single-view image. Therefore, we explicitly represent 3D shape structures as part relations and employ image supervision to guide the geometry and structure reconstruction. Trained with pairs of view-aligned images and 3D shapes, the VGSNet implicitly encodes the view-aware shape information in the latent feature space. Qualitative and quantitative comparisons with state-of-the-art baseline methods as well as ablation studies demonstrate the effectiveness of the VGSNet for structure-aware single-view 3D shape reconstruction.},
 bibtype = {article},
 author = {Zhang, Xuancheng and Ma, Rui and Zou, Changqing and Zhang, Minghao and Zhao, Xibin and Gao, Yue},
 doi = {10.1109/TPAMI.2021.3090917},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
 number = {c}
}

Reconstructing a 3D shape from a single-view image using deep learning is becoming increasingly popular recently. Most existing methods only focus on reconstructing the 3D shape geometry based on the image constraint. The lack of explicit modeling of structure relations among shape parts yields low-quality reconstruction results for structure-rich man-made shapes. In addition, conventional 2D-3D joint embedding architecture for image-based 3D shape reconstruction often omits the specific view information from the given image, which may lead to degraded geometry and structure reconstruction. We address these problems by introducing VGSNet, an encoder-decoder architecture for view-aware joint geometry and structure learning. The key idea is to jointly learn a multimodal feature representation of 2D image, 3D shape geometry and structure so that both geometry and structure details can be reconstructed from a single-view image. Therefore, we explicitly represent 3D shape structures as part relations and employ image supervision to guide the geometry and structure reconstruction. Trained with pairs of view-aligned images and 3D shapes, the VGSNet implicitly encodes the view-aware shape information in the latent feature space. Qualitative and quantitative comparisons with state-of-the-art baseline methods as well as ablation studies demonstrate the effectiveness of the VGSNet for structure-aware single-view 3D shape reconstruction.

@article{
 title = {Graph Spectral Point Cloud Processing},
 type = {article},
 year = {2021},
 id = {b89b300c-f749-319a-b946-02db72e1a9f1},
 created = {2021-09-30T06:29:37.911Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-17T16:08:04.883Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {c3a38ded-ec49-4494-8518-35cbd444f0c8,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Electric, Mitsubishi}
}

@article{
 title = {Walk in the Cloud: Learning Curves for Point Clouds Shape Analysis},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.01288},
 id = {3ef18a7b-1c3a-36d7-8b2f-bfec79f9a1e1},
 created = {2021-10-13T14:40:10.857Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:05.142Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Discrete point cloud objects lack sufficient shape descriptors of 3D geometries. In this paper, we present a novel method for aggregating hypothetical curves in point clouds. Sequences of connected points (curves) are initially grouped by taking guided walks in the point clouds, and then subsequently aggregated back to augment their point-wise features. We provide an effective implementation of the proposed aggregation strategy including a novel curve grouping operator followed by a curve aggregation operator. Our method was benchmarked on several point cloud analysis tasks where we achieved the state-of-the-art classification accuracy of 94.2% on the ModelNet40 classification task, instance IoU of 86.8 on the ShapeNetPart segmentation task, and cosine error of 0.11 on the ModelNet40 normal estimation task.},
 bibtype = {article},
 author = {Xiang, Tiange and Zhang, Chaoyi and Song, Yang and Yu, Jianhui and Cai, Weidong}
}

Discrete point cloud objects lack sufficient shape descriptors of 3D geometries. In this paper, we present a novel method for aggregating hypothetical curves in point clouds. Sequences of connected points (curves) are initially grouped by taking guided walks in the point clouds, and then subsequently aggregated back to augment their point-wise features. We provide an effective implementation of the proposed aggregation strategy including a novel curve grouping operator followed by a curve aggregation operator. Our method was benchmarked on several point cloud analysis tasks where we achieved the state-of-the-art classification accuracy of 94.2% on the ModelNet40 classification task, instance IoU of 86.8 on the ShapeNetPart segmentation task, and cosine error of 0.11 on the ModelNet40 normal estimation task.

@article{
 title = {Residual Attention: A Simple but Effective Method for Multi-Label Recognition},
 type = {article},
 year = {2021},
 pages = {184-193},
 websites = {http://arxiv.org/abs/2108.02456},
 id = {5dbaf040-be45-3356-9305-5ff5fe77f9fd},
 created = {2021-10-13T14:40:10.899Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:31.471Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Multi-label image recognition is a challenging computer vision task of practical use. Progresses in this area, however, are often characterized by complicated methods, heavy computations, and lack of intuitive explanations. To effectively capture different spatial regions occupied by objects from different categories, we propose an embarrassingly simple module, named class-specific residual attention (CSRA). CSRA generates class-specific features for every category by proposing a simple spatial attention score, and then combines it with the class-agnostic average pooling feature. CSRA achieves state-of-the-art results on multilabel recognition, and at the same time is much simpler than them. Furthermore, with only 4 lines of code, CSRA also leads to consistent improvement across many diverse pretrained models and datasets without any extra training. CSRA is both easy to implement and light in computations, which also enjoys intuitive explanations and visualizations.},
 bibtype = {article},
 author = {Zhu, Ke and Wu, Jianxin}
}

Multi-label image recognition is a challenging computer vision task of practical use. Progresses in this area, however, are often characterized by complicated methods, heavy computations, and lack of intuitive explanations. To effectively capture different spatial regions occupied by objects from different categories, we propose an embarrassingly simple module, named class-specific residual attention (CSRA). CSRA generates class-specific features for every category by proposing a simple spatial attention score, and then combines it with the class-agnostic average pooling feature. CSRA achieves state-of-the-art results on multilabel recognition, and at the same time is much simpler than them. Furthermore, with only 4 lines of code, CSRA also leads to consistent improvement across many diverse pretrained models and datasets without any extra training. CSRA is both easy to implement and light in computations, which also enjoys intuitive explanations and visualizations.

@article{
 title = {CvT: Introducing Convolutions to Vision Transformers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.15808},
 id = {1b0123e2-a5a9-3a9b-ad10-99f2ad23582b},
 created = {2021-10-13T14:40:10.925Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-13T14:44:02.890Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0},
 private_publication = {false},
 abstract = {We present in this paper a new architecture, named Convolutional vision Transformer (CvT), that improves Vision Transformer (ViT) in performance and efficiency by introducing convolutions into ViT to yield the best of both designs. This is accomplished through two primary modifications: a hierarchy of Transformers containing a new convolutional token embedding, and a convolutional Transformer block leveraging a convolutional projection. These changes introduce desirable properties of convolutional neural networks (CNNs) to the ViT architecture (\ie shift, scale, and distortion invariance) while maintaining the merits of Transformers (\ie dynamic attention, global context, and better generalization). We validate CvT by conducting extensive experiments, showing that this approach achieves state-of-the-art performance over other Vision Transformers and ResNets on ImageNet-1k, with fewer parameters and lower FLOPs. In addition, performance gains are maintained when pretrained on larger datasets (\eg ImageNet-22k) and fine-tuned to downstream tasks. Pre-trained on ImageNet-22k, our CvT-W24 obtains a top-1 accuracy of 87.7\% on the ImageNet-1k val set. Finally, our results show that the positional encoding, a crucial component in existing Vision Transformers, can be safely removed in our model, simplifying the design for higher resolution vision tasks. Code will be released at \urlhttps://github.com/leoxiaobin/CvT.},
 bibtype = {article},
 author = {Wu, Haiping and Xiao, Bin and Codella, Noel and Liu, Mengchen and Dai, Xiyang and Yuan, Lu and Zhang, Lei}
}

We present in this paper a new architecture, named Convolutional vision Transformer (CvT), that improves Vision Transformer (ViT) in performance and efficiency by introducing convolutions into ViT to yield the best of both designs. This is accomplished through two primary modifications: a hierarchy of Transformers containing a new convolutional token embedding, and a convolutional Transformer block leveraging a convolutional projection. These changes introduce desirable properties of convolutional neural networks (CNNs) to the ViT architecture (\ie shift, scale, and distortion invariance) while maintaining the merits of Transformers (\ie dynamic attention, global context, and better generalization). We validate CvT by conducting extensive experiments, showing that this approach achieves state-of-the-art performance over other Vision Transformers and ResNets on ImageNet-1k, with fewer parameters and lower FLOPs. In addition, performance gains are maintained when pretrained on larger datasets (\eg ImageNet-22k) and fine-tuned to downstream tasks. Pre-trained on ImageNet-22k, our CvT-W24 obtains a top-1 accuracy of 87.7\% on the ImageNet-1k val set. Finally, our results show that the positional encoding, a crucial component in existing Vision Transformers, can be safely removed in our model, simplifying the design for higher resolution vision tasks. Code will be released at \urlhttps://github.com/leoxiaobin/CvT.

@article{
 title = {Unsupervised Learning of Fine Structure Generation for 3D Point Clouds by 2D Projection Matching},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2108.03746},
 id = {d89c3431-6e58-3d4f-bd59-3679b6e37486},
 created = {2021-10-13T14:40:10.931Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:07.228Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Learning to generate 3D point clouds without 3D supervision is an important but challenging problem. Current solutions leverage various differentiable renderers to project the generated 3D point clouds onto a 2D image plane, and train deep neural networks using the per-pixel difference with 2D ground truth images. However, these solutions are still struggling to fully recover fine structures of 3D shapes, such as thin tubes or planes. To resolve this issue, we propose an unsupervised approach for 3D point cloud generation with fine structures. Specifically, we cast 3D point cloud learning as a 2D projection matching problem. Rather than using entire 2D silhouette images as a regular pixel supervision, we introduce structure adaptive sampling to randomly sample 2D points within the silhouettes as an irregular point supervision, which alleviates the consistency issue of sampling from different view angles. Our method pushes the neural network to generate a 3D point cloud whose 2D projections match the irregular point supervision from different view angles. Our 2D projection matching approach enables the neural network to learn more accurate structure information than using the per-pixel difference, especially for fine and thin 3D structures. Our method can recover fine 3D structures from 2D silhouette images at different resolutions, and is robust to different sampling methods and point number in irregular point supervision. Our method outperforms others under widely used benchmarks. Our code, data and models are available at https://github.com/chenchao15/2D\_projection\_matching.},
 bibtype = {article},
 author = {Chao, Chen and Han, Zhizhong and Liu, Yu-Shen and Zwicker, Matthias},
 number = {62072268}
}

Learning to generate 3D point clouds without 3D supervision is an important but challenging problem. Current solutions leverage various differentiable renderers to project the generated 3D point clouds onto a 2D image plane, and train deep neural networks using the per-pixel difference with 2D ground truth images. However, these solutions are still struggling to fully recover fine structures of 3D shapes, such as thin tubes or planes. To resolve this issue, we propose an unsupervised approach for 3D point cloud generation with fine structures. Specifically, we cast 3D point cloud learning as a 2D projection matching problem. Rather than using entire 2D silhouette images as a regular pixel supervision, we introduce structure adaptive sampling to randomly sample 2D points within the silhouettes as an irregular point supervision, which alleviates the consistency issue of sampling from different view angles. Our method pushes the neural network to generate a 3D point cloud whose 2D projections match the irregular point supervision from different view angles. Our 2D projection matching approach enables the neural network to learn more accurate structure information than using the per-pixel difference, especially for fine and thin 3D structures. Our method can recover fine 3D structures from 2D silhouette images at different resolutions, and is robust to different sampling methods and point number in irregular point supervision. Our method outperforms others under widely used benchmarks. Our code, data and models are available at https://github.com/chenchao15/2D\_projection\_matching.

@article{
 title = {Going deeper with Image Transformers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.17239},
 id = {cba87732-dece-395f-9797-82b349c9d5c7},
 created = {2021-10-13T14:40:10.932Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:31.805Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Transformers have been recently adapted for large scale image classification, achieving high scores shaking up the long supremacy of convolutional neural networks. However the optimization of image transformers has been little studied so far. In this work, we build and optimize deeper transformer networks for image classification. In particular, we investigate the interplay of architecture and optimization of such dedicated transformers. We make two transformers architecture changes that significantly improve the accuracy of deep transformers. This leads us to produce models whose performance does not saturate early with more depth, for instance we obtain 86.5% top-1 accuracy on Imagenet when training with no external data, we thus attain the current SOTA with less FLOPs and parameters. Moreover, our best model establishes the new state of the art on Imagenet with Reassessed labels and Imagenet-V2 / match frequency, in the setting with no additional training data. We share our code and models.},
 bibtype = {article},
 author = {Touvron, Hugo and Cord, Matthieu and Sablayrolles, Alexandre and Synnaeve, Gabriel and Jégou, Hervé}
}

Transformers have been recently adapted for large scale image classification, achieving high scores shaking up the long supremacy of convolutional neural networks. However the optimization of image transformers has been little studied so far. In this work, we build and optimize deeper transformer networks for image classification. In particular, we investigate the interplay of architecture and optimization of such dedicated transformers. We make two transformers architecture changes that significantly improve the accuracy of deep transformers. This leads us to produce models whose performance does not saturate early with more depth, for instance we obtain 86.5% top-1 accuracy on Imagenet when training with no external data, we thus attain the current SOTA with less FLOPs and parameters. Moreover, our best model establishes the new state of the art on Imagenet with Reassessed labels and Imagenet-V2 / match frequency, in the setting with no additional training data. We share our code and models.

@article{
 title = {Spatial-Temporal Transformer for Dynamic Scene Graph Generation},
 type = {article},
 year = {2021},
 pages = {16372-16382},
 websites = {http://arxiv.org/abs/2107.12309},
 id = {0d9d3c66-da3e-341c-9b83-386d384ea416},
 created = {2021-10-13T14:40:11.023Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:18.086Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Dynamic scene graph generation aims at generating a scene graph of the given video. Compared to the task of scene graph generation from images, it is more challenging because of the dynamic relationships between objects and the temporal dependencies between frames allowing for a richer semantic interpretation. In this paper, we propose Spatial-temporal Transformer (STTran), a neural network that consists of two core modules: (1) a spatial encoder that takes an input frame to extract spatial context and reason about the visual relationships within a frame, and (2) a temporal decoder which takes the output of the spatial encoder as input in order to capture the temporal dependencies between frames and infer the dynamic relationships. Furthermore, STTran is flexible to take varying lengths of videos as input without clipping, which is especially important for long videos. Our method is validated on the benchmark dataset Action Genome (AG). The experimental results demonstrate the superior performance of our method in terms of dynamic scene graphs. Moreover, a set of ablative studies is conducted and the effect of each proposed module is justified. Code available at: https://github.com/yrcong/STTran.},
 bibtype = {article},
 author = {Cong, Yuren and Liao, Wentong and Ackermann, Hanno and Rosenhahn, Bodo and Yang, Michael Ying}
}

Dynamic scene graph generation aims at generating a scene graph of the given video. Compared to the task of scene graph generation from images, it is more challenging because of the dynamic relationships between objects and the temporal dependencies between frames allowing for a richer semantic interpretation. In this paper, we propose Spatial-temporal Transformer (STTran), a neural network that consists of two core modules: (1) a spatial encoder that takes an input frame to extract spatial context and reason about the visual relationships within a frame, and (2) a temporal decoder which takes the output of the spatial encoder as input in order to capture the temporal dependencies between frames and infer the dynamic relationships. Furthermore, STTran is flexible to take varying lengths of videos as input without clipping, which is especially important for long videos. Our method is validated on the benchmark dataset Action Genome (AG). The experimental results demonstrate the superior performance of our method in terms of dynamic scene graphs. Moreover, a set of ablative studies is conducted and the effect of each proposed module is justified. Code available at: https://github.com/yrcong/STTran.

@article{
 title = {PointBA: Towards Backdoor Attacks in 3D Point Cloud},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.16074},
 id = {e3a16f07-52d0-3aa8-990c-277a1a2a4232},
 created = {2021-10-13T14:40:11.033Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:17.933Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {3D deep learning has been increasingly more popular for a variety of tasks including many safety-critical applications. However, recently several works raise the security issues of 3D deep models. Although most of them consider adversarial attacks, we identify that backdoor attack is indeed a more serious threat to 3D deep learning systems but remains unexplored. We present the backdoor attacks in 3D point cloud with a unified framework that exploits the unique properties of 3D data and networks. In particular, we design two attack approaches on point cloud: the poison-label backdoor attack (PointPBA) and the clean-label backdoor attack (PointCBA). The first one is straightforward and effective in practice, while the latter is more sophisticated assuming there are certain data inspections. The attack algorithms are mainly motivated and developed by 1) the recent discovery of 3D adversarial samples suggesting the vulnerability of deep models under spatial transformation; 2) the proposed feature disentanglement technique that manipulates the feature of the data through optimization methods and its potential to embed a new task. Extensive experiments show the efficacy of the PointPBA with over 95% success rate across various 3D datasets and models, and the more stealthy PointCBA with around 50% success rate. Our proposed backdoor attack in 3D point cloud is expected to perform as a baseline for improving the robustness of 3D deep models.},
 bibtype = {article},
 author = {Li, Xinke and Chen, Zhirui and Zhao, Yue and Tong, Zekun and Zhao, Yabang and Lim, Andrew and Zhou, Joey Tianyi}
}

3D deep learning has been increasingly more popular for a variety of tasks including many safety-critical applications. However, recently several works raise the security issues of 3D deep models. Although most of them consider adversarial attacks, we identify that backdoor attack is indeed a more serious threat to 3D deep learning systems but remains unexplored. We present the backdoor attacks in 3D point cloud with a unified framework that exploits the unique properties of 3D data and networks. In particular, we design two attack approaches on point cloud: the poison-label backdoor attack (PointPBA) and the clean-label backdoor attack (PointCBA). The first one is straightforward and effective in practice, while the latter is more sophisticated assuming there are certain data inspections. The attack algorithms are mainly motivated and developed by 1) the recent discovery of 3D adversarial samples suggesting the vulnerability of deep models under spatial transformation; 2) the proposed feature disentanglement technique that manipulates the feature of the data through optimization methods and its potential to embed a new task. Extensive experiments show the efficacy of the PointPBA with over 95% success rate across various 3D datasets and models, and the more stealthy PointCBA with around 50% success rate. Our proposed backdoor attack in 3D point cloud is expected to perform as a baseline for improving the robustness of 3D deep models.

@article{
 title = {Graph-to-3D: End-to-End Generation and Manipulation of 3D Scenes Using Scene Graphs},
 type = {article},
 year = {2021},
 pages = {16352-16361},
 websites = {http://arxiv.org/abs/2108.08841},
 id = {ff9ef537-b273-35d3-9fd4-e04296b06190},
 created = {2021-10-13T14:40:11.044Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:18.398Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Controllable scene synthesis consists of generating 3D information that satisfy underlying specifications. Thereby, these specifications should be abstract, i.e. allowing easy user interaction, whilst providing enough interface for detailed control. Scene graphs are representations of a scene, composed of objects (nodes) and inter-object relationships (edges), proven to be particularly suited for this task, as they allow for semantic control on the generated content. Previous works tackling this task often rely on synthetic data, and retrieve object meshes, which naturally limits the generation capabilities. To circumvent this issue, we instead propose the first work that directly generates shapes from a scene graph in an end-to-end manner. In addition, we show that the same model supports scene modification, using the respective scene graph as interface. Leveraging Graph Convolutional Networks (GCN) we train a variational Auto-Encoder on top of the object and edge categories, as well as 3D shapes and scene layouts, allowing latter sampling of new scenes and shapes.},
 bibtype = {article},
 author = {Dhamo, Helisa and Manhardt, Fabian and Navab, Nassir and Tombari, Federico}
}

Controllable scene synthesis consists of generating 3D information that satisfy underlying specifications. Thereby, these specifications should be abstract, i.e. allowing easy user interaction, whilst providing enough interface for detailed control. Scene graphs are representations of a scene, composed of objects (nodes) and inter-object relationships (edges), proven to be particularly suited for this task, as they allow for semantic control on the generated content. Previous works tackling this task often rely on synthetic data, and retrieve object meshes, which naturally limits the generation capabilities. To circumvent this issue, we instead propose the first work that directly generates shapes from a scene graph in an end-to-end manner. In addition, we show that the same model supports scene modification, using the respective scene graph as interface. Leveraging Graph Convolutional Networks (GCN) we train a variational Auto-Encoder on top of the object and edge categories, as well as 3D shapes and scene layouts, allowing latter sampling of new scenes and shapes.

@article{
 title = {Segmentation-grounded Scene Graph Generation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.14207},
 id = {e7d07e54-b11c-3cbc-877e-ef9cca391182},
 created = {2021-10-13T14:40:11.062Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:18.735Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Scene graph generation has emerged as an important problem in computer vision. While scene graphs provide a grounded representation of objects, their locations and relations in an image, they do so only at the granularity of proposal bounding boxes. In this work, we propose the first, to our knowledge, framework for pixel-level segmentation-grounded scene graph generation. Our framework is agnostic to the underlying scene graph generation method and address the lack of segmentation annotations in target scene graph datasets (e.g., Visual Genome) through transfer and multi-task learning from, and with, an auxiliary dataset (e.g., MS COCO). Specifically, each target object being detected is endowed with a segmentation mask, which is expressed as a lingual-similarity weighted linear combination over categories that have annotations present in an auxiliary dataset. These inferred masks, along with a novel Gaussian attention mechanism which grounds the relations at a pixel-level within the image, allow for improved relation prediction. The entire framework is end-to-end trainable and is learned in a multi-task manner with both target and auxiliary datasets.},
 bibtype = {article},
 author = {Khandelwal, Siddhesh and Suhail, Mohammed and Sigal, Leonid}
}

Scene graph generation has emerged as an important problem in computer vision. While scene graphs provide a grounded representation of objects, their locations and relations in an image, they do so only at the granularity of proposal bounding boxes. In this work, we propose the first, to our knowledge, framework for pixel-level segmentation-grounded scene graph generation. Our framework is agnostic to the underlying scene graph generation method and address the lack of segmentation annotations in target scene graph datasets (e.g., Visual Genome) through transfer and multi-task learning from, and with, an auxiliary dataset (e.g., MS COCO). Specifically, each target object being detected is endowed with a segmentation mask, which is expressed as a lingual-similarity weighted linear combination over categories that have annotations present in an auxiliary dataset. These inferred masks, along with a novel Gaussian attention mechanism which grounds the relations at a pixel-level within the image, allow for improved relation prediction. The entire framework is end-to-end trainable and is learned in a multi-task manner with both target and auxiliary datasets.

@article{
 title = {Unconditional Scene Graph Generation},
 type = {article},
 year = {2021},
 pages = {16362-16371},
 websites = {http://arxiv.org/abs/2108.05884},
 id = {cffaf4de-fb45-3d89-b446-6f8e73114eca},
 created = {2021-10-13T14:40:11.062Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:18.241Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Despite recent advancements in single-domain or single-object image generation, it is still challenging to generate complex scenes containing diverse, multiple objects and their interactions. Scene graphs, composed of nodes as objects and directed-edges as relationships among objects, offer an alternative representation of a scene that is more semantically grounded than images. We hypothesize that a generative model for scene graphs might be able to learn the underlying semantic structure of real-world scenes more effectively than images, and hence, generate realistic novel scenes in the form of scene graphs. In this work, we explore a new task for the unconditional generation of semantic scene graphs. We develop a deep auto-regressive model called SceneGraphGen which can directly learn the probability distribution over labelled and directed graphs using a hierarchical recurrent architecture. The model takes a seed object as input and generates a scene graph in a sequence of steps, each step generating an object node, followed by a sequence of relationship edges connecting to the previous nodes. We show that the scene graphs generated by SceneGraphGen are diverse and follow the semantic patterns of real-world scenes. Additionally, we demonstrate the application of the generated graphs in image synthesis, anomaly detection and scene graph completion.},
 bibtype = {article},
 author = {Garg, Sarthak and Dhamo, Helisa and Farshad, Azade and Musatian, Sabrina and Navab, Nassir and Tombari, Federico}
}

Despite recent advancements in single-domain or single-object image generation, it is still challenging to generate complex scenes containing diverse, multiple objects and their interactions. Scene graphs, composed of nodes as objects and directed-edges as relationships among objects, offer an alternative representation of a scene that is more semantically grounded than images. We hypothesize that a generative model for scene graphs might be able to learn the underlying semantic structure of real-world scenes more effectively than images, and hence, generate realistic novel scenes in the form of scene graphs. In this work, we explore a new task for the unconditional generation of semantic scene graphs. We develop a deep auto-regressive model called SceneGraphGen which can directly learn the probability distribution over labelled and directed graphs using a hierarchical recurrent architecture. The model takes a seed object as input and generates a scene graph in a sequence of steps, each step generating an object node, followed by a sequence of relationship edges connecting to the previous nodes. We show that the scene graphs generated by SceneGraphGen are diverse and follow the semantic patterns of real-world scenes. Additionally, we demonstrate the application of the generated graphs in image synthesis, anomaly detection and scene graph completion.

@article{
 title = {GANcraft: Unsupervised 3D Neural Rendering of Minecraft Worlds},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.07659},
 id = {3efaa1d8-6031-38b9-a32a-4cc55e8fdfb4},
 created = {2021-10-13T14:40:11.172Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:18.886Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present GANcraft, an unsupervised neural rendering framework for generating photorealistic images of large 3D block worlds such as those created in Minecraft. Our method takes a semantic block world as input, where each block is assigned a semantic label such as dirt, grass, or water. We represent the world as a continuous volumetric function and train our model to render view-consistent photorealistic images for a user-controlled camera. In the absence of paired ground truth real images for the block world, we devise a training technique based on pseudo-ground truth and adversarial training. This stands in contrast to prior work on neural rendering for view synthesis, which requires ground truth images to estimate scene geometry and view-dependent appearance. In addition to camera trajectory, GANcraft allows user control over both scene semantics and output style. Experimental results with comparison to strong baselines show the effectiveness of GANcraft on this novel task of photorealistic 3D block world synthesis. The project website is available at https://nvlabs.github.io/GANcraft/ .},
 bibtype = {article},
 author = {Hao, Zekun and Mallya, Arun and Belongie, Serge and Liu, Ming-Yu}
}

We present GANcraft, an unsupervised neural rendering framework for generating photorealistic images of large 3D block worlds such as those created in Minecraft. Our method takes a semantic block world as input, where each block is assigned a semantic label such as dirt, grass, or water. We represent the world as a continuous volumetric function and train our model to render view-consistent photorealistic images for a user-controlled camera. In the absence of paired ground truth real images for the block world, we devise a training technique based on pseudo-ground truth and adversarial training. This stands in contrast to prior work on neural rendering for view synthesis, which requires ground truth images to estimate scene geometry and view-dependent appearance. In addition to camera trajectory, GANcraft allows user control over both scene semantics and output style. Experimental results with comparison to strong baselines show the effectiveness of GANcraft on this novel task of photorealistic 3D block world synthesis. The project website is available at https://nvlabs.github.io/GANcraft/ .

@article{
 title = {Planar Surface Reconstruction from Sparse Views},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.14644},
 id = {2dd57164-5f56-39eb-bb4b-c0f5f1606df2},
 created = {2021-10-13T14:40:11.177Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:19.435Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {The paper studies planar surface reconstruction of indoor scenes from two views with unknown camera poses. While prior approaches have successfully created object-centric reconstructions of many scenes, they fail to exploit other structures, such as planes, which are typically the dominant components of indoor scenes. In this paper, we reconstruct planar surfaces from multiple views, while jointly estimating camera pose. Our experiments demonstrate that our method is able to advance the state of the art of reconstruction from sparse views, on challenging scenes from Matterport3D. Project site: https://jinlinyi.github.io/SparsePlanes/},
 bibtype = {article},
 author = {Jin, Linyi and Qian, Shengyi and Owens, Andrew and Fouhey, David F.}
}

The paper studies planar surface reconstruction of indoor scenes from two views with unknown camera poses. While prior approaches have successfully created object-centric reconstructions of many scenes, they fail to exploit other structures, such as planes, which are typically the dominant components of indoor scenes. In this paper, we reconstruct planar surfaces from multiple views, while jointly estimating camera pose. Our experiments demonstrate that our method is able to advance the state of the art of reconstruction from sparse views, on challenging scenes from Matterport3D. Project site: https://jinlinyi.github.io/SparsePlanes/

@article{
 title = {Sketch Your Own GAN},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2108.02774},
 id = {1be78aff-1d64-398b-ab73-ee0656c5f4d6},
 created = {2021-10-13T14:40:11.178Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:19.197Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Can a user create a deep generative model by sketching a single example? Traditionally, creating a GAN model has required the collection of a large-scale dataset of exemplars and specialized knowledge in deep learning. In contrast, sketching is possibly the most universally accessible way to convey a visual concept. In this work, we present a method, GAN Sketching, for rewriting GANs with one or more sketches, to make GANs training easier for novice users. In particular, we change the weights of an original GAN model according to user sketches. We encourage the model's output to match the user sketches through a cross-domain adversarial loss. Furthermore, we explore different regularization methods to preserve the original model's diversity and image quality. Experiments have shown that our method can mold GANs to match shapes and poses specified by sketches while maintaining realism and diversity. Finally, we demonstrate a few applications of the resulting GAN, including latent space interpolation and image editing.},
 bibtype = {article},
 author = {Wang, Sheng-Yu and Bau, David and Zhu, Jun-Yan}
}

Can a user create a deep generative model by sketching a single example? Traditionally, creating a GAN model has required the collection of a large-scale dataset of exemplars and specialized knowledge in deep learning. In contrast, sketching is possibly the most universally accessible way to convey a visual concept. In this work, we present a method, GAN Sketching, for rewriting GANs with one or more sketches, to make GANs training easier for novice users. In particular, we change the weights of an original GAN model according to user sketches. We encourage the model's output to match the user sketches through a cross-domain adversarial loss. Furthermore, we explore different regularization methods to preserve the original model's diversity and image quality. Experiments have shown that our method can mold GANs to match shapes and poses specified by sketches while maintaining realism and diversity. Finally, we demonstrate a few applications of the resulting GAN, including latent space interpolation and image editing.

@article{
 title = {Adaptive Surface Normal Constraint for Depth Estimation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.15483},
 id = {e2ab6aa5-ef77-3a1b-8348-6c27a3fead03},
 created = {2021-10-13T14:40:11.201Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:19.832Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present a novel method for single image depth estimation using surface normal constraints. Existing depth estimation methods either suffer from the lack of geometric constraints, or are limited to the difficulty of reliably capturing geometric context, which leads to a bottleneck of depth estimation quality. We therefore introduce a simple yet effective method, named Adaptive Surface Normal (ASN) constraint, to effectively correlate the depth estimation with geometric consistency. Our key idea is to adaptively determine the reliable local geometry from a set of randomly sampled candidates to derive surface normal constraint, for which we measure the consistency of the geometric contextual features. As a result, our method can faithfully reconstruct the 3D geometry and is robust to local shape variations, such as boundaries, sharp corners and noises. We conduct extensive evaluations and comparisons using public datasets. The experimental results demonstrate our method outperforms the state-of-the-art methods and has superior efficiency and robustness.},
 bibtype = {article},
 author = {Long, Xiaoxiao and Lin, Cheng and Liu, Lingjie and Li, Wei and Theobalt, Christian and Yang, Ruigang and Wang, Wenping}
}

We present a novel method for single image depth estimation using surface normal constraints. Existing depth estimation methods either suffer from the lack of geometric constraints, or are limited to the difficulty of reliably capturing geometric context, which leads to a bottleneck of depth estimation quality. We therefore introduce a simple yet effective method, named Adaptive Surface Normal (ASN) constraint, to effectively correlate the depth estimation with geometric consistency. Our key idea is to adaptively determine the reliable local geometry from a set of randomly sampled candidates to derive surface normal constraint, for which we measure the consistency of the geometric contextual features. As a result, our method can faithfully reconstruct the 3D geometry and is robust to local shape variations, such as boundaries, sharp corners and noises. We conduct extensive evaluations and comparisons using public datasets. The experimental results demonstrate our method outperforms the state-of-the-art methods and has superior efficiency and robustness.

@article{
 title = {Mesh Graphormer},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.00272},
 id = {3d903cc1-e9e1-35de-b6f6-e39d37d92fad},
 created = {2021-10-13T14:40:11.246Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:19.679Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present a graph-convolution-reinforced transformer, named Mesh Graphormer, for 3D human pose and mesh reconstruction from a single image. Recently both transformers and graph convolutional neural networks (GCNNs) have shown promising progress in human mesh reconstruction. Transformer-based approaches are effective in modeling non-local interactions among 3D mesh vertices and body joints, whereas GCNNs are good at exploiting neighborhood vertex interactions based on a pre-specified mesh topology. In this paper, we study how to combine graph convolutions and self-attentions in a transformer to model both local and global interactions. Experimental results show that our proposed method, Mesh Graphormer, significantly outperforms the previous state-of-the-art methods on multiple benchmarks, including Human3.6M, 3DPW, and FreiHAND datasets. Code and pre-trained models are available at https://github.com/microsoft/MeshGraphormer},
 bibtype = {article},
 author = {Lin, Kevin and Wang, Lijuan and Liu, Zicheng}
}

We present a graph-convolution-reinforced transformer, named Mesh Graphormer, for 3D human pose and mesh reconstruction from a single image. Recently both transformers and graph convolutional neural networks (GCNNs) have shown promising progress in human mesh reconstruction. Transformer-based approaches are effective in modeling non-local interactions among 3D mesh vertices and body joints, whereas GCNNs are good at exploiting neighborhood vertex interactions based on a pre-specified mesh topology. In this paper, we study how to combine graph convolutions and self-attentions in a transformer to model both local and global interactions. Experimental results show that our proposed method, Mesh Graphormer, significantly outperforms the previous state-of-the-art methods on multiple benchmarks, including Human3.6M, 3DPW, and FreiHAND datasets. Code and pre-trained models are available at https://github.com/microsoft/MeshGraphormer

@article{
 title = {PoinTr: Diverse Point Cloud Completion with Geometry-Aware Transformers},
 type = {article},
 year = {2021},
 pages = {12498-12507},
 websites = {http://arxiv.org/abs/2108.08839},
 id = {f19a6da0-7d56-38d8-afc2-c563317f608c},
 created = {2021-10-13T14:40:11.330Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:19.980Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Point clouds captured in real-world applications are often incomplete due to the limited sensor resolution, single viewpoint, and occlusion. Therefore, recovering the complete point clouds from partial ones becomes an indispensable task in many practical applications. In this paper, we present a new method that reformulates point cloud completion as a set-to-set translation problem and design a new model, called PoinTr that adopts a transformer encoder-decoder architecture for point cloud completion. By representing the point cloud as a set of unordered groups of points with position embeddings, we convert the point cloud to a sequence of point proxies and employ the transformers for point cloud generation. To facilitate transformers to better leverage the inductive bias about 3D geometric structures of point clouds, we further devise a geometry-aware block that models the local geometric relationships explicitly. The migration of transformers enables our model to better learn structural knowledge and preserve detailed information for point cloud completion. Furthermore, we propose two more challenging benchmarks with more diverse incomplete point clouds that can better reflect the real-world scenarios to promote future research. Experimental results show that our method outperforms state-of-the-art methods by a large margin on both the new benchmarks and the existing ones. Code is available at https://github.com/yuxumin/PoinTr},
 bibtype = {article},
 author = {Yu, Xumin and Rao, Yongming and Wang, Ziyi and Liu, Zuyan and Lu, Jiwen and Zhou, Jie}
}

Point clouds captured in real-world applications are often incomplete due to the limited sensor resolution, single viewpoint, and occlusion. Therefore, recovering the complete point clouds from partial ones becomes an indispensable task in many practical applications. In this paper, we present a new method that reformulates point cloud completion as a set-to-set translation problem and design a new model, called PoinTr that adopts a transformer encoder-decoder architecture for point cloud completion. By representing the point cloud as a set of unordered groups of points with position embeddings, we convert the point cloud to a sequence of point proxies and employ the transformers for point cloud generation. To facilitate transformers to better leverage the inductive bias about 3D geometric structures of point clouds, we further devise a geometry-aware block that models the local geometric relationships explicitly. The migration of transformers enables our model to better learn structural knowledge and preserve detailed information for point cloud completion. Furthermore, we propose two more challenging benchmarks with more diverse incomplete point clouds that can better reflect the real-world scenarios to promote future research. Experimental results show that our method outperforms state-of-the-art methods by a large margin on both the new benchmarks and the existing ones. Code is available at https://github.com/yuxumin/PoinTr

@article{
 title = {3DStyleNet: Creating 3D Shapes with Geometric and Texture Style Variations},
 type = {article},
 year = {2021},
 pages = {12456-12465},
 websites = {http://arxiv.org/abs/2108.12958},
 id = {3d1cac07-297a-3da9-ad88-abb4ac1e3c4a},
 created = {2021-10-13T14:40:11.337Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:20.152Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We propose a method to create plausible geometric and texture style variations of 3D objects in the quest to democratize 3D content creation. Given a pair of textured source and target objects, our method predicts a part-aware affine transformation field that naturally warps the source shape to imitate the overall geometric style of the target. In addition, the texture style of the target is transferred to the warped source object with the help of a multi-view differentiable renderer. Our model, 3DStyleNet, is composed of two sub-networks trained in two stages. First, the geometric style network is trained on a large set of untextured 3D shapes. Second, we jointly optimize our geometric style network and a pre-trained image style transfer network with losses defined over both the geometry and the rendering of the result. Given a small set of high-quality textured objects, our method can create many novel stylized shapes, resulting in effortless 3D content creation and style-ware data augmentation. We showcase our approach qualitatively on 3D content stylization, and provide user studies to validate the quality of our results. In addition, our method can serve as a valuable tool to create 3D data augmentations for computer vision tasks. Extensive quantitative analysis shows that 3DStyleNet outperforms alternative data augmentation techniques for the downstream task of single-image 3D reconstruction.},
 bibtype = {article},
 author = {Yin, Kangxue and Gao, Jun and Shugrina, Maria and Khamis, Sameh and Fidler, Sanja}
}

We propose a method to create plausible geometric and texture style variations of 3D objects in the quest to democratize 3D content creation. Given a pair of textured source and target objects, our method predicts a part-aware affine transformation field that naturally warps the source shape to imitate the overall geometric style of the target. In addition, the texture style of the target is transferred to the warped source object with the help of a multi-view differentiable renderer. Our model, 3DStyleNet, is composed of two sub-networks trained in two stages. First, the geometric style network is trained on a large set of untextured 3D shapes. Second, we jointly optimize our geometric style network and a pre-trained image style transfer network with losses defined over both the geometry and the rendering of the result. Given a small set of high-quality textured objects, our method can create many novel stylized shapes, resulting in effortless 3D content creation and style-ware data augmentation. We showcase our approach qualitatively on 3D content stylization, and provide user studies to validate the quality of our results. In addition, our method can serve as a valuable tool to create 3D data augmentations for computer vision tasks. Extensive quantitative analysis shows that 3DStyleNet outperforms alternative data augmentation techniques for the downstream task of single-image 3D reconstruction.

@article{
 title = {3DIAS: 3D Shape Reconstruction with Implicit Algebraic Surfaces},
 type = {article},
 year = {2021},
 pages = {12446-12455},
 websites = {http://arxiv.org/abs/2108.08653},
 id = {1aeb822c-b86e-33d3-a592-8d8b50af9a33},
 created = {2021-10-13T14:40:11.343Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:20.301Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {3D Shape representation has substantial effects on 3D shape reconstruction. Primitive-based representations approximate a 3D shape mainly by a set of simple implicit primitives, but the low geometrical complexity of the primitives limits the shape resolution. Moreover, setting a sufficient number of primitives for an arbitrary shape is challenging. To overcome these issues, we propose a constrained implicit algebraic surface as the primitive with few learnable coefficients and higher geometrical complexities and a deep neural network to produce these primitives. Our experiments demonstrate the superiorities of our method in terms of representation power compared to the state-of-the-art methods in single RGB image 3D shape reconstruction. Furthermore, we show that our method can semantically learn segments of 3D shapes in an unsupervised manner. The code is publicly available from https://myavartanoo.github.io/3dias/ .},
 bibtype = {article},
 author = {Yavartanoo, Mohsen and Chung, JaeYoung and Neshatavar, Reyhaneh and Lee, Kyoung Mu}
}

3D Shape representation has substantial effects on 3D shape reconstruction. Primitive-based representations approximate a 3D shape mainly by a set of simple implicit primitives, but the low geometrical complexity of the primitives limits the shape resolution. Moreover, setting a sufficient number of primitives for an arbitrary shape is challenging. To overcome these issues, we propose a constrained implicit algebraic surface as the primitive with few learnable coefficients and higher geometrical complexities and a deep neural network to produce these primitives. Our experiments demonstrate the superiorities of our method in terms of representation power compared to the state-of-the-art methods in single RGB image 3D shape reconstruction. Furthermore, we show that our method can semantically learn segments of 3D shapes in an unsupervised manner. The code is publicly available from https://myavartanoo.github.io/3dias/ .

@article{
 title = {Unsupervised Point Cloud Object Co-segmentation by Co-contrastive Learning and Mutual Attention Sampling},
 type = {article},
 year = {2021},
 pages = {1-10},
 id = {d61b01e7-789c-3b21-84fc-e0f31111b26d},
 created = {2021-10-13T14:40:11.366Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:07.487Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Cvpr, Anonymous and Id, Paper}
}

@article{
 title = {Manifold Matching via Deep Metric Learning for Generative Modeling},
 type = {article},
 year = {2021},
 pages = {6587-6597},
 websites = {http://arxiv.org/abs/2106.10777},
 id = {8f08bcac-d408-351b-bfff-82a27458514e},
 created = {2021-10-13T14:40:11.402Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:23.464Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We propose a manifold matching approach to generative models which includes a distribution generator (or data generator) and a metric generator. In our framework, we view the real data set as some manifold embedded in a high-dimensional Euclidean space. The distribution generator aims at generating samples that follow some distribution condensed around the real data manifold. It is achieved by matching two sets of points using their geometric shape descriptors, such as centroid and $p$-diameter, with learned distance metric; the metric generator utilizes both real data and generated samples to learn a distance metric which is close to some intrinsic geodesic distance on the real data manifold. The produced distance metric is further used for manifold matching. The two networks are learned simultaneously during the training process. We apply the approach on both unsupervised and supervised learning tasks: in unconditional image generation task, the proposed method obtains competitive results compared with existing generative models; in super-resolution task, we incorporate the framework in perception-based models and improve visual qualities by producing samples with more natural textures. Both theoretical analysis and real data experiments demonstrate the feasibility and effectiveness of the proposed framework.},
 bibtype = {article},
 author = {Dai, Mengyu and Hang, Haibin}
}

We propose a manifold matching approach to generative models which includes a distribution generator (or data generator) and a metric generator. In our framework, we view the real data set as some manifold embedded in a high-dimensional Euclidean space. The distribution generator aims at generating samples that follow some distribution condensed around the real data manifold. It is achieved by matching two sets of points using their geometric shape descriptors, such as centroid and $p$-diameter, with learned distance metric; the metric generator utilizes both real data and generated samples to learn a distance metric which is close to some intrinsic geodesic distance on the real data manifold. The produced distance metric is further used for manifold matching. The two networks are learned simultaneously during the training process. We apply the approach on both unsupervised and supervised learning tasks: in unconditional image generation task, the proposed method obtains competitive results compared with existing generative models; in super-resolution task, we incorporate the framework in perception-based models and improve visual qualities by producing samples with more natural textures. Both theoretical analysis and real data experiments demonstrate the feasibility and effectiveness of the proposed framework.

@article{
 title = {When do GANs replicate ? On the choice of dataset size},
 type = {article},
 year = {2021},
 pages = {6701-6710},
 id = {c3650cca-3fd2-380f-b8a8-585f84e855ba},
 created = {2021-10-13T14:40:11.465Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-19T07:34:03.926Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {b6d75013-efe2-4ddc-b3db-65496bd4db9f,a6a80a30-e9a2-486d-8032-eac3fd981996,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Feng, Qianli and Benitez-quiroz, Chenqi Guo Fabian}
}

@article{
 title = {Spatio-temporal Self-Supervised Representation Learning for 3D Point Clouds},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2109.00179},
 id = {61246011-bcfb-3c36-8207-8aba7ed96b0e},
 created = {2021-10-13T14:40:11.479Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:23.618Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {To date, various 3D scene understanding tasks still lack practical and generalizable pre-trained models, primarily due to the intricate nature of 3D scene understanding tasks and their immense variations introduced by camera views, lighting, occlusions, etc. In this paper, we tackle this challenge by introducing a spatio-temporal representation learning (STRL) framework, capable of learning from unlabeled 3D point clouds in a self-supervised fashion. Inspired by how infants learn from visual data in the wild, we explore the rich spatio-temporal cues derived from the 3D data. Specifically, STRL takes two temporally-correlated frames from a 3D point cloud sequence as the input, transforms it with the spatial data augmentation, and learns the invariant representation self-supervisedly. To corroborate the efficacy of STRL, we conduct extensive experiments on three types (synthetic, indoor, and outdoor) of datasets. Experimental results demonstrate that, compared with supervised learning methods, the learned self-supervised representation facilitates various models to attain comparable or even better performances while capable of generalizing pre-trained models to downstream tasks, including 3D shape classification, 3D object detection, and 3D semantic segmentation. Moreover, the spatio-temporal contextual cues embedded in 3D point clouds significantly improve the learned representations.},
 bibtype = {article},
 author = {Huang, Siyuan and Xie, Yichen and Zhu, Song-Chun and Zhu, Yixin}
}

To date, various 3D scene understanding tasks still lack practical and generalizable pre-trained models, primarily due to the intricate nature of 3D scene understanding tasks and their immense variations introduced by camera views, lighting, occlusions, etc. In this paper, we tackle this challenge by introducing a spatio-temporal representation learning (STRL) framework, capable of learning from unlabeled 3D point clouds in a self-supervised fashion. Inspired by how infants learn from visual data in the wild, we explore the rich spatio-temporal cues derived from the 3D data. Specifically, STRL takes two temporally-correlated frames from a 3D point cloud sequence as the input, transforms it with the spatial data augmentation, and learns the invariant representation self-supervisedly. To corroborate the efficacy of STRL, we conduct extensive experiments on three types (synthetic, indoor, and outdoor) of datasets. Experimental results demonstrate that, compared with supervised learning methods, the learned self-supervised representation facilitates various models to attain comparable or even better performances while capable of generalizing pre-trained models to downstream tasks, including 3D shape classification, 3D object detection, and 3D semantic segmentation. Moreover, the spatio-temporal contextual cues embedded in 3D point clouds significantly improve the learned representations.

@article{
 title = {Vis2Mesh: Efficient Mesh Reconstruction from Unstructured Point Clouds of Large Scenes with Learned Virtual View Visibility},
 type = {article},
 year = {2021},
 pages = {6514-6524},
 websites = {http://arxiv.org/abs/2108.08378},
 id = {f6da9524-1a84-3683-a414-cd8db7be64ef},
 created = {2021-10-13T14:40:11.499Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:54.845Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present a novel framework for mesh reconstruction from unstructured point clouds by taking advantage of the learned visibility of the 3D points in the virtual views and traditional graph-cut based mesh generation. Specifically, we first propose a three-step network that explicitly employs depth completion for visibility prediction. Then the visibility information of multiple views is aggregated to generate a 3D mesh model by solving an optimization problem considering visibility in which a novel adaptive visibility weighting in surface determination is also introduced to suppress line of sight with a large incident angle. Compared to other learning-based approaches, our pipeline only exercises the learning on a 2D binary classification task, \ie, points visible or not in a view, which is much more generalizable and practically more efficient and capable to deal with a large number of points. Experiments demonstrate that our method with favorable transferability and robustness, and achieve competing performances \wrt state-of-the-art learning-based approaches on small complex objects and outperforms on large indoor and outdoor scenes. Code is available at https://github.com/GDAOSU/vis2mesh.},
 bibtype = {article},
 author = {Song, Shuang and Cui, Zhaopeng and Qin, Rongjun}
}

We present a novel framework for mesh reconstruction from unstructured point clouds by taking advantage of the learned visibility of the 3D points in the virtual views and traditional graph-cut based mesh generation. Specifically, we first propose a three-step network that explicitly employs depth completion for visibility prediction. Then the visibility information of multiple views is aggregated to generate a 3D mesh model by solving an optimization problem considering visibility in which a novel adaptive visibility weighting in surface determination is also introduced to suppress line of sight with a large incident angle. Compared to other learning-based approaches, our pipeline only exercises the learning on a 2D binary classification task, \ie, points visible or not in a view, which is much more generalizable and practically more efficient and capable to deal with a large number of points. Experiments demonstrate that our method with favorable transferability and robustness, and achieve competing performances \wrt state-of-the-art learning-based approaches on small complex objects and outperforms on large indoor and outdoor scenes. Code is available at https://github.com/GDAOSU/vis2mesh.

@article{
 title = {Learning Signed Distance Field for Multi-view Surface Reconstruction},
 type = {article},
 year = {2021},
 pages = {6525-6534},
 websites = {http://arxiv.org/abs/2108.09964},
 id = {ac722251-2f3d-3c01-b4be-5c16ae6a67fe},
 created = {2021-10-13T14:40:11.504Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:23.759Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Recent works on implicit neural representations have shown promising results for multi-view surface reconstruction. However, most approaches are limited to relatively simple geometries and usually require clean object masks for reconstructing complex and concave objects. In this work, we introduce a novel neural surface reconstruction framework that leverages the knowledge of stereo matching and feature consistency to optimize the implicit surface representation. More specifically, we apply a signed distance field (SDF) and a surface light field to represent the scene geometry and appearance respectively. The SDF is directly supervised by geometry from stereo matching, and is refined by optimizing the multi-view feature consistency and the fidelity of rendered images. Our method is able to improve the robustness of geometry estimation and support reconstruction of complex scene topologies. Extensive experiments have been conducted on DTU, EPFL and Tanks and Temples datasets. Compared to previous state-of-the-art methods, our method achieves better mesh reconstruction in wide open scenes without masks as input.},
 bibtype = {article},
 author = {Zhang, Jingyang and Yao, Yao and Quan, Long}
}

Recent works on implicit neural representations have shown promising results for multi-view surface reconstruction. However, most approaches are limited to relatively simple geometries and usually require clean object masks for reconstructing complex and concave objects. In this work, we introduce a novel neural surface reconstruction framework that leverages the knowledge of stereo matching and feature consistency to optimize the implicit surface representation. More specifically, we apply a signed distance field (SDF) and a surface light field to represent the scene geometry and appearance respectively. The SDF is directly supervised by geometry from stereo matching, and is refined by optimizing the multi-view feature consistency and the fidelity of rendered images. Our method is able to improve the robustness of geometry estimation and support reconstruction of complex scene topologies. Extensive experiments have been conducted on DTU, EPFL and Tanks and Temples datasets. Compared to previous state-of-the-art methods, our method achieves better mesh reconstruction in wide open scenes without masks as input.

@article{
 title = {Multi-view 3D Reconstruction with Transformer},
 type = {article},
 year = {2021},
 pages = {1-14},
 websites = {http://arxiv.org/abs/2103.12957},
 id = {6b0820aa-5571-3399-9548-13843ce867f0},
 created = {2021-10-13T14:40:11.593Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:07.165Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wang2021},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b,cd02f564-0123-4236-a320-b339927f085a},
 private_publication = {false},
 abstract = {Deep CNN-based methods have so far achieved the state of the art results in multi-view 3D object reconstruction. Despite the considerable progress, the two core modules of these methods - multi-view feature extraction and fusion, are usually investigated separately, and the object relations in different views are rarely explored. In this paper, inspired by the recent great success in self-attention-based Transformer models, we reformulate the multi-view 3D reconstruction as a sequence-to-sequence prediction problem and propose a new framework named 3D Volume Transformer (VolT) for such a task. Unlike previous CNN-based methods using a separate design, we unify the feature extraction and view fusion in a single Transformer network. A natural advantage of our design lies in the exploration of view-to-view relationships using self-attention among multiple unordered inputs. On ShapeNet - a large-scale 3D reconstruction benchmark dataset, our method achieves a new state-of-the-art accuracy in multi-view reconstruction with fewer parameters ($70\%$ less) than other CNN-based methods. Experimental results also suggest the strong scaling capability of our method. Our code will be made publicly available.},
 bibtype = {article},
 author = {Wang, Dan and Cui, Xinrui and Chen, Xun and Zou, Zhengxia and Shi, Tianyang and Salcudean, Septimiu and Wang, Z. Jane and Ward, Rabab}
}

Deep CNN-based methods have so far achieved the state of the art results in multi-view 3D object reconstruction. Despite the considerable progress, the two core modules of these methods - multi-view feature extraction and fusion, are usually investigated separately, and the object relations in different views are rarely explored. In this paper, inspired by the recent great success in self-attention-based Transformer models, we reformulate the multi-view 3D reconstruction as a sequence-to-sequence prediction problem and propose a new framework named 3D Volume Transformer (VolT) for such a task. Unlike previous CNN-based methods using a separate design, we unify the feature extraction and view fusion in a single Transformer network. A natural advantage of our design lies in the exploration of view-to-view relationships using self-attention among multiple unordered inputs. On ShapeNet - a large-scale 3D reconstruction benchmark dataset, our method achieves a new state-of-the-art accuracy in multi-view reconstruction with fewer parameters ($70\%$ less) than other CNN-based methods. Experimental results also suggest the strong scaling capability of our method. Our code will be made publicly available.

@article{
 title = {Adaptive Graph Convolution for Point Cloud Analysis},
 type = {article},
 year = {2021},
 pages = {4965-4974},
 websites = {http://arxiv.org/abs/2108.08035},
 id = {9888bd0b-81e0-3872-9e2c-e7245e07e20f},
 created = {2021-10-13T14:40:11.612Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:06.311Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Convolution on 3D point clouds that generalized from 2D grid-like domains is widely researched yet far from perfect. The standard convolution characterises feature correspondences indistinguishably among 3D points, presenting an intrinsic limitation of poor distinctive feature learning. In this paper, we propose Adaptive Graph Convolution (AdaptConv) which generates adaptive kernels for points according to their dynamically learned features. Compared with using a fixed/isotropic kernel, AdaptConv improves the flexibility of point cloud convolutions, effectively and precisely capturing the diverse relations between points from different semantic parts. Unlike popular attentional weight schemes, the proposed AdaptConv implements the adaptiveness inside the convolution operation instead of simply assigning different weights to the neighboring points. Extensive qualitative and quantitative evaluations show that our method outperforms state-of-the-art point cloud classification and segmentation approaches on several benchmark datasets. Our code is available at https://github.com/hrzhou2/AdaptConv-master.},
 bibtype = {article},
 author = {Zhou, Haoran and Feng, Yidan and Fang, Mingsheng and Wei, Mingqiang and Qin, Jing and Lu, Tong}
}

Convolution on 3D point clouds that generalized from 2D grid-like domains is widely researched yet far from perfect. The standard convolution characterises feature correspondences indistinguishably among 3D points, presenting an intrinsic limitation of poor distinctive feature learning. In this paper, we propose Adaptive Graph Convolution (AdaptConv) which generates adaptive kernels for points according to their dynamically learned features. Compared with using a fixed/isotropic kernel, AdaptConv improves the flexibility of point cloud convolutions, effectively and precisely capturing the diverse relations between points from different semantic parts. Unlike popular attentional weight schemes, the proposed AdaptConv implements the adaptiveness inside the convolution operation instead of simply assigning different weights to the neighboring points. Extensive qualitative and quantitative evaluations show that our method outperforms state-of-the-art point cloud classification and segmentation approaches on several benchmark datasets. Our code is available at https://github.com/hrzhou2/AdaptConv-master.

@article{
 title = {Deep Structured Instance Graph for Distilling Object Detectors},
 type = {article},
 year = {2021},
 pages = {4359-4368},
 websites = {http://arxiv.org/abs/2109.12862},
 id = {cc3215d0-97c8-3b74-a43c-a9626dd0556d},
 created = {2021-10-13T14:40:11.622Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:24.738Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Effectively structuring deep knowledge plays a pivotal role in transfer from teacher to student, especially in semantic vision tasks. In this paper, we present a simple knowledge structure to exploit and encode information inside the detection system to facilitate detector knowledge distillation. Specifically, aiming at solving the feature imbalance problem while further excavating the missing relation inside semantic instances, we design a graph whose nodes correspond to instance proposal-level features and edges represent the relation between nodes. To further refine this graph, we design an adaptive background loss weight to reduce node noise and background samples mining to prune trivial edges. We transfer the entire graph as encoded knowledge representation from teacher to student, capturing local and global information simultaneously. We achieve new state-of-the-art results on the challenging COCO object detection task with diverse student-teacher pairs on both one- and two-stage detectors. We also experiment with instance segmentation to demonstrate robustness of our method. It is notable that distilled Faster R-CNN with ResNet18-FPN and ResNet50-FPN yields 38.68 and 41.82 Box AP respectively on the COCO benchmark, Faster R-CNN with ResNet101-FPN significantly achieves 43.38 AP, which outperforms ResNet152-FPN teacher about 0.7 AP. Code: https://github.com/dvlab-research/Dsig.},
 bibtype = {article},
 author = {Chen, Yixin and Chen, Pengguang and Liu, Shu and Wang, Liwei and Jia, Jiaya}
}

Effectively structuring deep knowledge plays a pivotal role in transfer from teacher to student, especially in semantic vision tasks. In this paper, we present a simple knowledge structure to exploit and encode information inside the detection system to facilitate detector knowledge distillation. Specifically, aiming at solving the feature imbalance problem while further excavating the missing relation inside semantic instances, we design a graph whose nodes correspond to instance proposal-level features and edges represent the relation between nodes. To further refine this graph, we design an adaptive background loss weight to reduce node noise and background samples mining to prune trivial edges. We transfer the entire graph as encoded knowledge representation from teacher to student, capturing local and global information simultaneously. We achieve new state-of-the-art results on the challenging COCO object detection task with diverse student-teacher pairs on both one- and two-stage detectors. We also experiment with instance segmentation to demonstrate robustness of our method. It is notable that distilled Faster R-CNN with ResNet18-FPN and ResNet50-FPN yields 38.68 and 41.82 Box AP respectively on the COCO benchmark, Faster R-CNN with ResNet101-FPN significantly achieves 43.38 AP, which outperforms ResNet152-FPN teacher about 0.7 AP. Code: https://github.com/dvlab-research/Dsig.

@article{
 title = {Dynamic Attentive Graph Learning for Image Restoration},
 type = {article},
 year = {2021},
 pages = {4328-4337},
 websites = {http://arxiv.org/abs/2109.06620},
 id = {bd1667fa-4ecf-301a-8442-f19394f0d11f},
 created = {2021-10-13T14:40:11.629Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:06.052Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Non-local self-similarity in natural images has been verified to be an effective prior for image restoration. However, most existing deep non-local methods assign a fixed number of neighbors for each query item, neglecting the dynamics of non-local correlations. Moreover, the non-local correlations are usually based on pixels, prone to be biased due to image degradation. To rectify these weaknesses, in this paper, we propose a dynamic attentive graph learning model (DAGL) to explore the dynamic non-local property on patch level for image restoration. Specifically, we propose an improved graph model to perform patch-wise graph convolution with a dynamic and adaptive number of neighbors for each node. In this way, image content can adaptively balance over-smooth and over-sharp artifacts through the number of its connected neighbors, and the patch-wise non-local correlations can enhance the message passing process. Experimental results on various image restoration tasks: synthetic image denoising, real image denoising, image demosaicing, and compression artifact reduction show that our DAGL can produce state-of-the-art results with superior accuracy and visual quality. The source code is available at https://github.com/jianzhangcs/DAGL.},
 bibtype = {article},
 author = {Mou, Chong and Zhang, Jian and Wu, Zhuoyuan}
}

Non-local self-similarity in natural images has been verified to be an effective prior for image restoration. However, most existing deep non-local methods assign a fixed number of neighbors for each query item, neglecting the dynamics of non-local correlations. Moreover, the non-local correlations are usually based on pixels, prone to be biased due to image degradation. To rectify these weaknesses, in this paper, we propose a dynamic attentive graph learning model (DAGL) to explore the dynamic non-local property on patch level for image restoration. Specifically, we propose an improved graph model to perform patch-wise graph convolution with a dynamic and adaptive number of neighbors for each node. In this way, image content can adaptively balance over-smooth and over-sharp artifacts through the number of its connected neighbors, and the patch-wise non-local correlations can enhance the message passing process. Experimental results on various image restoration tasks: synthetic image denoising, real image denoising, image demosaicing, and compression artifact reduction show that our DAGL can produce state-of-the-art results with superior accuracy and visual quality. The source code is available at https://github.com/jianzhangcs/DAGL.

@article{
 title = {RGB-D Saliency Detection via Cascaded Mutual Information Minimization},
 type = {article},
 year = {2021},
 pages = {4338-4347},
 websites = {http://arxiv.org/abs/2109.07246},
 id = {3d2fc1af-57f4-35d7-92c8-f76854e5157e},
 created = {2021-10-13T14:40:11.643Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:55.447Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Existing RGB-D saliency detection models do not explicitly encourage RGB and depth to achieve effective multi-modal learning. In this paper, we introduce a novel multi-stage cascaded learning framework via mutual information minimization to "explicitly" model the multi-modal information between RGB image and depth data. Specifically, we first map the feature of each mode to a lower dimensional feature vector, and adopt mutual information minimization as a regularizer to reduce the redundancy between appearance features from RGB and geometric features from depth. We then perform multi-stage cascaded learning to impose the mutual information minimization constraint at every stage of the network. Extensive experiments on benchmark RGB-D saliency datasets illustrate the effectiveness of our framework. Further, to prosper the development of this field, we contribute the largest (7x larger than NJU2K) dataset, which contains 15,625 image pairs with high quality polygon-/scribble-/object-/instance-/rank-level annotations. Based on these rich labels, we additionally construct four new benchmarks with strong baselines and observe some interesting phenomena, which can motivate future model design. Source code and dataset are available at "https://github.com/JingZhang617/cascaded_rgbd_sod".},
 bibtype = {article},
 author = {Zhang, Jing and Fan, Deng-Ping and Dai, Yuchao and Yu, Xin and Zhong, Yiran and Barnes, Nick and Shao, Ling}
}

Existing RGB-D saliency detection models do not explicitly encourage RGB and depth to achieve effective multi-modal learning. In this paper, we introduce a novel multi-stage cascaded learning framework via mutual information minimization to "explicitly" model the multi-modal information between RGB image and depth data. Specifically, we first map the feature of each mode to a lower dimensional feature vector, and adopt mutual information minimization as a regularizer to reduce the redundancy between appearance features from RGB and geometric features from depth. We then perform multi-stage cascaded learning to impose the mutual information minimization constraint at every stage of the network. Extensive experiments on benchmark RGB-D saliency datasets illustrate the effectiveness of our framework. Further, to prosper the development of this field, we contribute the largest (7x larger than NJU2K) dataset, which contains 15,625 image pairs with high quality polygon-/scribble-/object-/instance-/rank-level annotations. Based on these rich labels, we additionally construct four new benchmarks with strong baselines and observe some interesting phenomena, which can motivate future model design. Source code and dataset are available at "https://github.com/JingZhang617/cascaded_rgbd_sod".

@article{
 title = {Topologically Consistent Multi-View Face Inference Using Volumetric Sampling},
 type = {article},
 year = {2021},
 pages = {3824-3834},
 id = {301f8b0c-cb4f-3dfd-acaf-67a294aa8bc2},
 created = {2021-10-13T14:40:11.731Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:25.216Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {High-fidelity face digitization solutions often combine multi-view stereo (MVS) techniques for 3D reconstruction and a non-rigid registration step to establish dense correspondence across identities and expressions. A common problem is the need for manual clean-up after the MVS step, as 3D scans are typically affected by noise and outliers and contain hairy surface regions that need to be cleaned up by artists. Furthermore, mesh registration tends to fail for extreme facial expressions. Most learning-based methods use an underlying 3D morphable model (3DMM) to ensure robustness, but this limits the output accuracy for extreme facial expressions. In addition, the global bottleneck of regression architectures cannot produce meshes that tightly fit the ground truth surfaces. We propose ToFu, Topologically consistent Face from multi-view, a geometry inference framework that can produce topologically consistent meshes across facial identities and expressions using a volumetric representation instead of an explicit underlying 3DMM. Our novel progressive mesh generation network embeds the topological structure of the face in a feature volume, sampled from geometry-aware local features. A coarse-to-fine architecture facilitates dense and accurate facial mesh predictions in a consistent mesh topology. ToFu further captures displacement maps for pore-level geometric details and facilitates high-quality rendering in the form of albedo and specular reflectance maps. These high-quality assets are readily usable by production studios for avatar creation, animation and physically-based skin rendering. We demonstrate state-of-the-art geometric and correspondence accuracy, while only taking 0.385 seconds to compute a mesh with 10K vertices, which is three orders of magnitude faster than traditional techniques. The code and the model are available for research purposes at https://tianyeli.github.io/tofu.},
 bibtype = {article},
 author = {Iccv, Anonymous and Id, Paper}
}

High-fidelity face digitization solutions often combine multi-view stereo (MVS) techniques for 3D reconstruction and a non-rigid registration step to establish dense correspondence across identities and expressions. A common problem is the need for manual clean-up after the MVS step, as 3D scans are typically affected by noise and outliers and contain hairy surface regions that need to be cleaned up by artists. Furthermore, mesh registration tends to fail for extreme facial expressions. Most learning-based methods use an underlying 3D morphable model (3DMM) to ensure robustness, but this limits the output accuracy for extreme facial expressions. In addition, the global bottleneck of regression architectures cannot produce meshes that tightly fit the ground truth surfaces. We propose ToFu, Topologically consistent Face from multi-view, a geometry inference framework that can produce topologically consistent meshes across facial identities and expressions using a volumetric representation instead of an explicit underlying 3DMM. Our novel progressive mesh generation network embeds the topological structure of the face in a feature volume, sampled from geometry-aware local features. A coarse-to-fine architecture facilitates dense and accurate facial mesh predictions in a consistent mesh topology. ToFu further captures displacement maps for pore-level geometric details and facilitates high-quality rendering in the form of albedo and specular reflectance maps. These high-quality assets are readily usable by production studios for avatar creation, animation and physically-based skin rendering. We demonstrate state-of-the-art geometric and correspondence accuracy, while only taking 0.385 seconds to compute a mesh with 10K vertices, which is three orders of magnitude faster than traditional techniques. The code and the model are available for research purposes at https://tianyeli.github.io/tofu.

@article{
 title = {Point-set Distances for Learning Representations of 3D Point Clouds},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2102.04014},
 id = {bbd9f1a0-06c0-3aa2-a353-493a45c07efb},
 created = {2021-10-13T14:40:11.744Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:06.953Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Learning an effective representation of 3D point clouds requires a good metric to measure the discrepancy between two 3D point sets, which is non-trivial due to their irregularity. Most of the previous works resort to using the Chamfer discrepancy or Earth Mover's distance, but those metrics are either ineffective in measuring the differences between point clouds or computationally expensive. In this paper, we conduct a systematic study with extensive experiments on distance metrics for 3D point clouds. From this study, we propose to use a variant of the Wasserstein distance, named the sliced Wasserstein distance, for learning representations of 3D point clouds. Experiments show that the sliced Wasserstein distance allows the neural network to learn a more efficient representation compared to the Chamfer discrepancy. We demonstrate the efficiency of the sliced Wasserstein metric on several tasks in 3D computer vision including training a point cloud autoencoder, generative modeling, transfer learning, and point cloud registration.},
 bibtype = {article},
 author = {Nguyen, Trung and Pham, Quang-Hieu and Le, Tam and Pham, Tung and Ho, Nhat and Hua, Binh-Son},
 number = {Section 4}
}

Learning an effective representation of 3D point clouds requires a good metric to measure the discrepancy between two 3D point sets, which is non-trivial due to their irregularity. Most of the previous works resort to using the Chamfer discrepancy or Earth Mover's distance, but those metrics are either ineffective in measuring the differences between point clouds or computationally expensive. In this paper, we conduct a systematic study with extensive experiments on distance metrics for 3D point clouds. From this study, we propose to use a variant of the Wasserstein distance, named the sliced Wasserstein distance, for learning representations of 3D point clouds. Experiments show that the sliced Wasserstein distance allows the neural network to learn a more efficient representation compared to the Chamfer discrepancy. We demonstrate the efficiency of the sliced Wasserstein metric on several tasks in 3D computer vision including training a point cloud autoencoder, generative modeling, transfer learning, and point cloud registration.

@article{
 title = {Self-Supervised Pretraining of 3D Features on any Point-Cloud},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2101.02691},
 id = {f0c3c8a4-22f8-3ae0-be06-ca2c470a5b0b},
 created = {2021-10-13T14:40:11.759Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:20.924Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Pretraining on large labeled datasets is a prerequisite to achieve good performance in many computer vision tasks like 2D object recognition, video classification etc. However, pretraining is not widely used for 3D recognition tasks where state-of-the-art methods train models from scratch. A primary reason is the lack of large annotated datasets because 3D data is both difficult to acquire and time consuming to label. We present a simple self-supervised pertaining method that can work with any 3D data - single or multiview, indoor or outdoor, acquired by varied sensors, without 3D registration. We pretrain standard point cloud and voxel based model architectures, and show that joint pretraining further improves performance. We evaluate our models on 9 benchmarks for object detection, semantic segmentation, and object classification, where they achieve state-of-the-art results and can outperform supervised pretraining. We set a new state-of-the-art for object detection on ScanNet (69.0% mAP) and SUNRGBD (63.5% mAP). Our pretrained models are label efficient and improve performance for classes with few examples.},
 bibtype = {article},
 author = {Zhang, Zaiwei and Girdhar, Rohit and Joulin, Armand and Misra, Ishan}
}

Pretraining on large labeled datasets is a prerequisite to achieve good performance in many computer vision tasks like 2D object recognition, video classification etc. However, pretraining is not widely used for 3D recognition tasks where state-of-the-art methods train models from scratch. A primary reason is the lack of large annotated datasets because 3D data is both difficult to acquire and time consuming to label. We present a simple self-supervised pertaining method that can work with any 3D data - single or multiview, indoor or outdoor, acquired by varied sensors, without 3D registration. We pretrain standard point cloud and voxel based model architectures, and show that joint pretraining further improves performance. We evaluate our models on 9 benchmarks for object detection, semantic segmentation, and object classification, where they achieve state-of-the-art results and can outperform supervised pretraining. We set a new state-of-the-art for object detection on ScanNet (69.0% mAP) and SUNRGBD (63.5% mAP). Our pretrained models are label efficient and improve performance for classes with few examples.

@article{
 title = {Active Learning for Deep Object Detection via Probabilistic Modeling},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.16130},
 id = {f675e0ad-60ea-3e42-840e-1d41266ca4f6},
 created = {2021-10-13T14:40:11.762Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:20.766Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Active learning aims to reduce labeling costs by selecting only the most informative samples on a dataset. Few existing works have addressed active learning for object detection. Most of these methods are based on multiple models or are straightforward extensions of classification methods, hence estimate an image's informativeness using only the classification head. In this paper, we propose a novel deep active learning approach for object detection. Our approach relies on mixture density networks that estimate a probabilistic distribution for each localization and classification head's output. We explicitly estimate the aleatoric and epistemic uncertainty in a single forward pass of a single model. Our method uses a scoring function that aggregates these two types of uncertainties for both heads to obtain every image's informativeness score. We demonstrate the efficacy of our approach in PASCAL VOC and MS-COCO datasets. Our approach outperforms single-model based methods and performs on par with multi-model based methods at a fraction of the computing cost.},
 bibtype = {article},
 author = {Choi, Jiwoong and Elezi, Ismail and Lee, Hyuk-Jae and Farabet, Clement and Alvarez, Jose M.}
}

Active learning aims to reduce labeling costs by selecting only the most informative samples on a dataset. Few existing works have addressed active learning for object detection. Most of these methods are based on multiple models or are straightforward extensions of classification methods, hence estimate an image's informativeness using only the classification head. In this paper, we propose a novel deep active learning approach for object detection. Our approach relies on mixture density networks that estimate a probabilistic distribution for each localization and classification head's output. We explicitly estimate the aleatoric and epistemic uncertainty in a single forward pass of a single model. Our method uses a scoring function that aggregates these two types of uncertainties for both heads to obtain every image's informativeness score. We demonstrate the efficacy of our approach in PASCAL VOC and MS-COCO datasets. Our approach outperforms single-model based methods and performs on par with multi-model based methods at a fraction of the computing cost.

@article{
 title = {Common Objects in 3D: Large-Scale Learning and Evaluation of Real-life 3D Category Reconstruction},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2109.00512},
 id = {9ac6fc54-d9a0-35cd-8d4f-efef3e6919ca},
 created = {2021-10-13T14:40:11.765Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:54.645Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Traditional approaches for learning 3D object categories have been predominantly trained and evaluated on synthetic datasets due to the unavailability of real 3D-annotated category-centric data. Our main goal is to facilitate advances in this field by collecting real-world data in a magnitude similar to the existing synthetic counterparts. The principal contribution of this work is thus a large-scale dataset, called Common Objects in 3D, with real multi-view images of object categories annotated with camera poses and ground truth 3D point clouds. The dataset contains a total of 1.5 million frames from nearly 19,000 videos capturing objects from 50 MS-COCO categories and, as such, it is significantly larger than alternatives both in terms of the number of categories and objects. We exploit this new dataset to conduct one of the first large-scale "in-the-wild" evaluations of several new-view-synthesis and category-centric 3D reconstruction methods. Finally, we contribute NerFormer - a novel neural rendering method that leverages the powerful Transformer to reconstruct an object given a small number of its views. The CO3D dataset is available at https://github.com/facebookresearch/co3d .},
 bibtype = {article},
 author = {Reizenstein, Jeremy and Shapovalov, Roman and Henzler, Philipp and Sbordone, Luca and Labatut, Patrick and Novotny, David}
}

Traditional approaches for learning 3D object categories have been predominantly trained and evaluated on synthetic datasets due to the unavailability of real 3D-annotated category-centric data. Our main goal is to facilitate advances in this field by collecting real-world data in a magnitude similar to the existing synthetic counterparts. The principal contribution of this work is thus a large-scale dataset, called Common Objects in 3D, with real multi-view images of object categories annotated with camera poses and ground truth 3D point clouds. The dataset contains a total of 1.5 million frames from nearly 19,000 videos capturing objects from 50 MS-COCO categories and, as such, it is significantly larger than alternatives both in terms of the number of categories and objects. We exploit this new dataset to conduct one of the first large-scale "in-the-wild" evaluations of several new-view-synthesis and category-centric 3D reconstruction methods. Finally, we contribute NerFormer - a novel neural rendering method that leverages the powerful Transformer to reconstruct an object given a small number of its views. The CO3D dataset is available at https://github.com/facebookresearch/co3d .

@article{
 title = {Exploiting a Joint Embedding Space for Generalized Zero-Shot Semantic Segmentation},
 type = {article},
 year = {2021},
 pages = {9536-9545},
 websites = {http://arxiv.org/abs/2108.06536},
 id = {e8d65bfb-9709-374c-9143-3290046630f9},
 created = {2021-10-13T14:40:11.833Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:21.237Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We address the problem of generalized zero-shot semantic segmentation (GZS3) predicting pixel-wise semantic labels for seen and unseen classes. Most GZS3 methods adopt a generative approach that synthesizes visual features of unseen classes from corresponding semantic ones (e.g., word2vec) to train novel classifiers for both seen and unseen classes. Although generative methods show decent performance, they have two limitations: (1) the visual features are biased towards seen classes; (2) the classifier should be retrained whenever novel unseen classes appear. We propose a discriminative approach to address these limitations in a unified framework. To this end, we leverage visual and semantic encoders to learn a joint embedding space, where the semantic encoder transforms semantic features to semantic prototypes that act as centers for visual features of corresponding classes. Specifically, we introduce boundary-aware regression (BAR) and semantic consistency (SC) losses to learn discriminative features. Our approach to exploiting the joint embedding space, together with BAR and SC terms, alleviates the seen bias problem. At test time, we avoid the retraining process by exploiting semantic prototypes as a nearest-neighbor (NN) classifier. To further alleviate the bias problem, we also propose an inference technique, dubbed Apollonius calibration (AC), that modulates the decision boundary of the NN classifier to the Apollonius circle adaptively. Experimental results demonstrate the effectiveness of our framework, achieving a new state of the art on standard benchmarks.},
 bibtype = {article},
 author = {Baek, Donghyeon and Oh, Youngmin and Ham, Bumsub}
}

We address the problem of generalized zero-shot semantic segmentation (GZS3) predicting pixel-wise semantic labels for seen and unseen classes. Most GZS3 methods adopt a generative approach that synthesizes visual features of unseen classes from corresponding semantic ones (e.g., word2vec) to train novel classifiers for both seen and unseen classes. Although generative methods show decent performance, they have two limitations: (1) the visual features are biased towards seen classes; (2) the classifier should be retrained whenever novel unseen classes appear. We propose a discriminative approach to address these limitations in a unified framework. To this end, we leverage visual and semantic encoders to learn a joint embedding space, where the semantic encoder transforms semantic features to semantic prototypes that act as centers for visual features of corresponding classes. Specifically, we introduce boundary-aware regression (BAR) and semantic consistency (SC) losses to learn discriminative features. Our approach to exploiting the joint embedding space, together with BAR and SC terms, alleviates the seen bias problem. At test time, we avoid the retraining process by exploiting semantic prototypes as a nearest-neighbor (NN) classifier. To further alleviate the bias problem, we also propose an inference technique, dubbed Apollonius calibration (AC), that modulates the decision boundary of the NN classifier to the Apollonius circle adaptively. Experimental results demonstrate the effectiveness of our framework, achieving a new state of the art on standard benchmarks.

@article{
 title = {Adaptive Adversarial Network for Source-free Domain Adaptation},
 type = {article},
 year = {2021},
 pages = {9010-9019},
 id = {8dbe0c8b-629f-3d1c-bc7c-a408661ad59d},
 created = {2021-10-13T14:40:11.882Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:21.423Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Iccv, Anonymous and Id, Paper}
}

@article{
 title = {Simpler is Better: Few-shot Semantic Segmentation with Classifier Weight Transformer},
 type = {article},
 year = {2021},
 pages = {8741-8750},
 websites = {http://arxiv.org/abs/2108.03032},
 id = {2e54dbd3-7d01-33e9-a1fd-03be62a55fc5},
 created = {2021-10-13T14:40:11.900Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:21.761Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {A few-shot semantic segmentation model is typically composed of a CNN encoder, a CNN decoder and a simple classifier (separating foreground and background pixels). Most existing methods meta-learn all three model components for fast adaptation to a new class. However, given that as few as a single support set image is available, effective model adaption of all three components to the new class is extremely challenging. In this work we propose to simplify the meta-learning task by focusing solely on the simplest component, the classifier, whilst leaving the encoder and decoder to pre-training. We hypothesize that if we pre-train an off-the-shelf segmentation model over a set of diverse training classes with sufficient annotations, the encoder and decoder can capture rich discriminative features applicable for any unseen classes, rendering the subsequent meta-learning stage unnecessary. For the classifier meta-learning, we introduce a Classifier Weight Transformer (CWT) designed to dynamically adapt the supportset trained classifier's weights to each query image in an inductive way. Extensive experiments on two standard benchmarks show that despite its simplicity, our method outperforms the state-of-the-art alternatives, often by a large margin.Code is available on https://github.com/zhiheLu/CWT-for-FSS.},
 bibtype = {article},
 author = {Lu, Zhihe and He, Sen and Zhu, Xiatian and Zhang, Li and Song, Yi-Zhe and Xiang, Tao}
}

A few-shot semantic segmentation model is typically composed of a CNN encoder, a CNN decoder and a simple classifier (separating foreground and background pixels). Most existing methods meta-learn all three model components for fast adaptation to a new class. However, given that as few as a single support set image is available, effective model adaption of all three components to the new class is extremely challenging. In this work we propose to simplify the meta-learning task by focusing solely on the simplest component, the classifier, whilst leaving the encoder and decoder to pre-training. We hypothesize that if we pre-train an off-the-shelf segmentation model over a set of diverse training classes with sufficient annotations, the encoder and decoder can capture rich discriminative features applicable for any unseen classes, rendering the subsequent meta-learning stage unnecessary. For the classifier meta-learning, we introduce a Classifier Weight Transformer (CWT) designed to dynamically adapt the supportset trained classifier's weights to each query image in an inductive way. Extensive experiments on two standard benchmarks show that despite its simplicity, our method outperforms the state-of-the-art alternatives, often by a large margin.Code is available on https://github.com/zhiheLu/CWT-for-FSS.

@article{
 title = {DeFRCN: Decoupled Faster R-CNN for Few-Shot Object Detection},
 type = {article},
 year = {2021},
 pages = {8681-8690},
 websites = {http://arxiv.org/abs/2108.09017},
 id = {bcce9b6a-8b07-3de4-91d8-5ed66baee5af},
 created = {2021-10-13T14:40:11.995Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:22.078Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Few-shot object detection, which aims at detecting novel objects rapidly from extremely few annotated examples of previously unseen classes, has attracted significant research interest in the community. Most existing approaches employ the Faster R-CNN as basic detection framework, yet, due to the lack of tailored considerations for data-scarce scenario, their performance is often not satisfactory. In this paper, we look closely into the conventional Faster R-CNN and analyze its contradictions from two orthogonal perspectives, namely multi-stage (RPN vs. RCNN) and multi-task (classification vs. localization). To resolve these issues, we propose a simple yet effective architecture, named Decoupled Faster R-CNN (DeFRCN). To be concrete, we extend Faster R-CNN by introducing Gradient Decoupled Layer for multi-stage decoupling and Prototypical Calibration Block for multi-task decoupling. The former is a novel deep layer with redefining the feature-forward operation and gradient-backward operation for decoupling its subsequent layer and preceding layer, and the latter is an offline prototype-based classification model with taking the proposals from detector as input and boosting the original classification scores with additional pairwise scores for calibration. Extensive experiments on multiple benchmarks show our framework is remarkably superior to other existing approaches and establishes a new state-of-the-art in few-shot literature.},
 bibtype = {article},
 author = {Qiao, Limeng and Zhao, Yuxuan and Li, Zhiyuan and Qiu, Xi and Wu, Jianan and Zhang, Chi}
}

Few-shot object detection, which aims at detecting novel objects rapidly from extremely few annotated examples of previously unseen classes, has attracted significant research interest in the community. Most existing approaches employ the Faster R-CNN as basic detection framework, yet, due to the lack of tailored considerations for data-scarce scenario, their performance is often not satisfactory. In this paper, we look closely into the conventional Faster R-CNN and analyze its contradictions from two orthogonal perspectives, namely multi-stage (RPN vs. RCNN) and multi-task (classification vs. localization). To resolve these issues, we propose a simple yet effective architecture, named Decoupled Faster R-CNN (DeFRCN). To be concrete, we extend Faster R-CNN by introducing Gradient Decoupled Layer for multi-stage decoupling and Prototypical Calibration Block for multi-task decoupling. The former is a novel deep layer with redefining the feature-forward operation and gradient-backward operation for decoupling its subsequent layer and preceding layer, and the latter is an offline prototype-based classification model with taking the proposals from detector as input and boosting the original classification scores with additional pairwise scores for calibration. Extensive experiments on multiple benchmarks show our framework is remarkably superior to other existing approaches and establishes a new state-of-the-art in few-shot literature.

@article{
 title = {Fooling LiDAR Perception via Adversarial Trajectory Perturbation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.15326},
 id = {bde982d3-c47c-3f7d-80ed-e3e77a738e95},
 created = {2021-10-13T14:40:12.005Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:22.396Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {LiDAR point clouds collected from a moving vehicle are functions of its trajectories, because the sensor motion needs to be compensated to avoid distortions. When autonomous vehicles are sending LiDAR point clouds to deep networks for perception and planning, could the motion compensation consequently become a wide-open backdoor in those networks, due to both the adversarial vulnerability of deep learning and GPS-based vehicle trajectory estimation that is susceptible to wireless spoofing? We demonstrate such possibilities for the first time: instead of directly attacking point cloud coordinates which requires tampering with the raw LiDAR readings, only adversarial spoofing of a self-driving car's trajectory with small perturbations is enough to make safety-critical objects undetectable or detected with incorrect positions. Moreover, polynomial trajectory perturbation is developed to achieve a temporally-smooth and highly-imperceptible attack. Extensive experiments on 3D object detection have shown that such attacks not only lower the performance of the state-of-the-art detectors effectively, but also transfer to other detectors, raising a red flag for the community. The code is available on https://ai4ce.github.io/FLAT/.},
 bibtype = {article},
 author = {Li, Yiming and Wen, Congcong and Juefei-Xu, Felix and Feng, Chen}
}

LiDAR point clouds collected from a moving vehicle are functions of its trajectories, because the sensor motion needs to be compensated to avoid distortions. When autonomous vehicles are sending LiDAR point clouds to deep networks for perception and planning, could the motion compensation consequently become a wide-open backdoor in those networks, due to both the adversarial vulnerability of deep learning and GPS-based vehicle trajectory estimation that is susceptible to wireless spoofing? We demonstrate such possibilities for the first time: instead of directly attacking point cloud coordinates which requires tampering with the raw LiDAR readings, only adversarial spoofing of a self-driving car's trajectory with small perturbations is enough to make safety-critical objects undetectable or detected with incorrect positions. Moreover, polynomial trajectory perturbation is developed to achieve a temporally-smooth and highly-imperceptible attack. Extensive experiments on 3D object detection have shown that such attacks not only lower the performance of the state-of-the-art detectors effectively, but also transfer to other detectors, raising a red flag for the community. The code is available on https://ai4ce.github.io/FLAT/.

@article{
 title = {Differentiable Convolution Search for Point Cloud Processing},
 type = {article},
 year = {2021},
 pages = {7437-7446},
 websites = {http://arxiv.org/abs/2108.12856},
 id = {93444432-2744-30a2-aa10-b6f2000db176},
 created = {2021-10-13T14:40:12.027Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-21T12:52:20.641Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Exploiting convolutional neural networks for point cloud processing is quite challenging, due to the inherent irregular distribution and discrete shape representation of point clouds. To address these problems, many handcrafted convolution variants have sprung up in recent years. Though with elaborate design, these variants could be far from optimal in sufficiently capturing diverse shapes formed by discrete points. In this paper, we propose PointSeaConv, i.e., a novel differential convolution search paradigm on point clouds. It can work in a purely data-driven manner and thus is capable of auto-creating a group of suitable convolutions for geometric shape modeling. We also propose a joint optimization framework for simultaneous search of internal convolution and external architecture, and introduce epsilon-greedy algorithm to alleviate the effect of discretization error. As a result, PointSeaNet, a deep network that is sufficient to capture geometric shapes at both convolution level and architecture level, can be searched out for point cloud processing. Extensive experiments strongly evidence that our proposed PointSeaNet surpasses current handcrafted deep models on challenging benchmarks across multiple tasks with remarkable margins.},
 bibtype = {article},
 author = {Nie, Xing and Liu, Yongcheng and Chen, Shaohong and Chang, Jianlong and Huo, Chunlei and Meng, Gaofeng and Tian, Qi and Hu, Weiming and Pan, Chunhong}
}

Exploiting convolutional neural networks for point cloud processing is quite challenging, due to the inherent irregular distribution and discrete shape representation of point clouds. To address these problems, many handcrafted convolution variants have sprung up in recent years. Though with elaborate design, these variants could be far from optimal in sufficiently capturing diverse shapes formed by discrete points. In this paper, we propose PointSeaConv, i.e., a novel differential convolution search paradigm on point clouds. It can work in a purely data-driven manner and thus is capable of auto-creating a group of suitable convolutions for geometric shape modeling. We also propose a joint optimization framework for simultaneous search of internal convolution and external architecture, and introduce epsilon-greedy algorithm to alleviate the effect of discretization error. As a result, PointSeaNet, a deep network that is sufficient to capture geometric shapes at both convolution level and architecture level, can be searched out for point cloud processing. Extensive experiments strongly evidence that our proposed PointSeaNet surpasses current handcrafted deep models on challenging benchmarks across multiple tasks with remarkable margins.

@article{
 title = {PR-GCN: A Deep Graph Convolutional Network with Point Refinement for 6D Pose Estimation},
 type = {article},
 year = {2021},
 pages = {2793-2802},
 websites = {http://arxiv.org/abs/2108.09916},
 id = {a9715c52-ae0a-3a9e-9962-8976d6b1a64e},
 created = {2021-10-13T14:40:12.081Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:05.454Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {RGB-D based 6D pose estimation has recently achieved remarkable progress, but still suffers from two major limitations: (1) ineffective representation of depth data and (2) insufficient integration of different modalities. This paper proposes a novel deep learning approach, namely Graph Convolutional Network with Point Refinement (PR-GCN), to simultaneously address the issues above in a unified way. It first introduces the Point Refinement Network (PRN) to polish 3D point clouds, recovering missing parts with noise removed. Subsequently, the Multi-Modal Fusion Graph Convolutional Network (MMF-GCN) is presented to strengthen RGB-D combination, which captures geometry-aware inter-modality correlation through local information propagation in the graph convolutional network. Extensive experiments are conducted on three widely used benchmarks, and state-of-the-art performance is reached. Besides, it is also shown that the proposed PRN and MMF-GCN modules are well generalized to other frameworks.},
 bibtype = {article},
 author = {Zhou, Guangyuan and Wang, Huiqun and Chen, Jiaxin and Huang, Di}
}

RGB-D based 6D pose estimation has recently achieved remarkable progress, but still suffers from two major limitations: (1) ineffective representation of depth data and (2) insufficient integration of different modalities. This paper proposes a novel deep learning approach, namely Graph Convolutional Network with Point Refinement (PR-GCN), to simultaneously address the issues above in a unified way. It first introduces the Point Refinement Network (PRN) to polish 3D point clouds, recovering missing parts with noise removed. Subsequently, the Multi-Modal Fusion Graph Convolutional Network (MMF-GCN) is presented to strengthen RGB-D combination, which captures geometry-aware inter-modality correlation through local information propagation in the graph convolutional network. Extensive experiments are conducted on three widely used benchmarks, and state-of-the-art performance is reached. Besides, it is also shown that the proposed PRN and MMF-GCN modules are well generalized to other frameworks.

@article{
 title = {Weakly Supervised 3D Semantic Segmentation Using Cross-Image Consensus and Inter-Voxel Affinity Relations},
 type = {article},
 year = {2021},
 pages = {2834-2844},
 id = {3eaf10ef-e4f7-31b7-a88c-7e15c1fe7e14},
 created = {2021-10-13T14:40:12.132Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:28.126Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Cvpr, Anonymous and Id, Paper}
}

@article{
 title = {SGPA : Structure-Guided Prior Adaptation for Category-Level 6D Object Pose Estimation},
 type = {article},
 year = {2021},
 pages = {2773-2782},
 id = {229a2413-e909-3abd-bdf0-4f70debed722},
 created = {2021-10-13T14:40:12.150Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:28.585Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Iccv, Iccv}
}

@article{
 title = {HPNet: Deep Primitive Segmentation Using Hybrid Representations},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.10620},
 id = {9ba1f30f-09a1-3231-867f-fecdd4f639cd},
 created = {2021-10-13T14:40:12.169Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-21T12:52:20.633Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {This paper introduces HPNet, a novel deep-learning approach for segmenting a 3D shape represented as a point cloud into primitive patches. The key to deep primitive segmentation is learning a feature representation that can separate points of different primitives. Unlike utilizing a single feature representation, HPNet leverages hybrid representations that combine one learned semantic descriptor, two spectral descriptors derived from predicted geometric parameters, as well as an adjacency matrix that encodes sharp edges. Moreover, instead of merely concatenating the descriptors, HPNet optimally combines hybrid representations by learning combination weights. This weighting module builds on the entropy of input features. The output primitive segmentation is obtained from a mean-shift clustering module. Experimental results on benchmark datasets ANSI and ABCParts show that HPNet leads to significant performance gains from baseline approaches.},
 bibtype = {article},
 author = {Yan, Siming and Yang, Zhenpei and Ma, Chongyang and Huang, Haibin and Vouga, Etienne and Huang, Qixing}
}

This paper introduces HPNet, a novel deep-learning approach for segmenting a 3D shape represented as a point cloud into primitive patches. The key to deep primitive segmentation is learning a feature representation that can separate points of different primitives. Unlike utilizing a single feature representation, HPNet leverages hybrid representations that combine one learned semantic descriptor, two spectral descriptors derived from predicted geometric parameters, as well as an adjacency matrix that encodes sharp edges. Moreover, instead of merely concatenating the descriptors, HPNet optimally combines hybrid representations by learning combination weights. This weighting module builds on the entropy of input features. The output primitive segmentation is obtained from a mean-shift clustering module. Experimental results on benchmark datasets ANSI and ABCParts show that HPNet leads to significant performance gains from baseline approaches.

@article{
 title = {Instance Segmentation in 3D Scenes using Semantic Superpoint Tree Networks},
 type = {article},
 year = {2021},
 pages = {2783-2792},
 websites = {http://arxiv.org/abs/2108.07478},
 id = {c86ca77f-04c3-3098-9aa5-8b08bd73bd51},
 created = {2021-10-13T14:40:12.187Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:28.430Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Instance segmentation in 3D scenes is fundamental in many applications of scene understanding. It is yet challenging due to the compound factors of data irregularity and uncertainty in the numbers of instances. State-of-the-art methods largely rely on a general pipeline that first learns point-wise features discriminative at semantic and instance levels, followed by a separate step of point grouping for proposing object instances. While promising, they have the shortcomings that (1) the second step is not supervised by the main objective of instance segmentation, and (2) their point-wise feature learning and grouping are less effective to deal with data irregularities, possibly resulting in fragmented segmentations. To address these issues, we propose in this work an end-to-end solution of Semantic Superpoint Tree Network (SSTNet) for proposing object instances from scene points. Key in SSTNet is an intermediate, semantic superpoint tree (SST), which is constructed based on the learned semantic features of superpoints, and which will be traversed and split at intermediate tree nodes for proposals of object instances. We also design in SSTNet a refinement module, termed CliqueNet, to prune superpoints that may be wrongly grouped into instance proposals. Experiments on the benchmarks of ScanNet and S3DIS show the efficacy of our proposed method. At the time of submission, SSTNet ranks top on the ScanNet (V2) leaderboard, with 2% higher of mAP than the second best method. The source code in PyTorch is available at https://github.com/Gorilla-Lab-SCUT/SSTNet.},
 bibtype = {article},
 author = {Liang, Zhihao and Li, Zhihao and Xu, Songcen and Tan, Mingkui and Jia, Kui}
}

Instance segmentation in 3D scenes is fundamental in many applications of scene understanding. It is yet challenging due to the compound factors of data irregularity and uncertainty in the numbers of instances. State-of-the-art methods largely rely on a general pipeline that first learns point-wise features discriminative at semantic and instance levels, followed by a separate step of point grouping for proposing object instances. While promising, they have the shortcomings that (1) the second step is not supervised by the main objective of instance segmentation, and (2) their point-wise feature learning and grouping are less effective to deal with data irregularities, possibly resulting in fragmented segmentations. To address these issues, we propose in this work an end-to-end solution of Semantic Superpoint Tree Network (SSTNet) for proposing object instances from scene points. Key in SSTNet is an intermediate, semantic superpoint tree (SST), which is constructed based on the learned semantic features of superpoints, and which will be traversed and split at intermediate tree nodes for proposals of object instances. We also design in SSTNet a refinement module, termed CliqueNet, to prune superpoints that may be wrongly grouped into instance proposals. Experiments on the benchmarks of ScanNet and S3DIS show the efficacy of our proposed method. At the time of submission, SSTNet ranks top on the ScanNet (V2) leaderboard, with 2% higher of mAP than the second best method. The source code in PyTorch is available at https://github.com/Gorilla-Lab-SCUT/SSTNet.

@article{
 title = {Learning Multi-Scene Absolute Pose Regression with Transformers},
 type = {article},
 year = {2021},
 pages = {4-7},
 websites = {http://arxiv.org/abs/2103.11468},
 id = {e6bf5004-3733-3d39-87c2-3ca0984a2cb1},
 created = {2021-10-13T14:40:12.228Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.190Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Absolute camera pose regressors estimate the position and orientation of a camera from the captured image alone. Typically, a convolutional backbone with a multi-layer perceptron head is trained with images and pose labels to embed a single reference scene at a time. Recently, this scheme was extended for learning multiple scenes by replacing the MLP head with a set of fully connected layers. In this work, we propose to learn multi-scene absolute camera pose regression with Transformers, where encoders are used to aggregate activation maps with self-attention and decoders transform latent features and scenes encoding into candidate pose predictions. This mechanism allows our model to focus on general features that are informative for localization while embedding multiple scenes in parallel. We evaluate our method on commonly benchmarked indoor and outdoor datasets and show that it surpasses both multi-scene and state-of-the-art single-scene absolute pose regressors. We make our code publicly available from https://github.com/yolish/multi-scene-pose-transformer.},
 bibtype = {article},
 author = {Shavit, Yoli and Ferens, Ron and Keller, Yosi}
}

Absolute camera pose regressors estimate the position and orientation of a camera from the captured image alone. Typically, a convolutional backbone with a multi-layer perceptron head is trained with images and pose labels to embed a single reference scene at a time. Recently, this scheme was extended for learning multiple scenes by replacing the MLP head with a set of fully connected layers. In this work, we propose to learn multi-scene absolute camera pose regression with Transformers, where encoders are used to aggregate activation maps with self-attention and decoders transform latent features and scenes encoding into candidate pose predictions. This mechanism allows our model to focus on general features that are informative for localization while embedding multiple scenes in parallel. We evaluate our method on commonly benchmarked indoor and outdoor datasets and show that it surpasses both multi-scene and state-of-the-art single-scene absolute pose regressors. We make our code publicly available from https://github.com/yolish/multi-scene-pose-transformer.

@article{
 title = {Improving 3D Object Detection with Channel-wise Transformer},
 type = {article},
 year = {2021},
 pages = {2743-2752},
 websites = {http://arxiv.org/abs/2108.10723},
 id = {61263b20-597e-3e81-a929-90df13b64c30},
 created = {2021-10-13T14:40:12.254Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-13T14:43:23.125Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0},
 private_publication = {false},
 abstract = {Though 3D object detection from point clouds has achieved rapid progress in recent years, the lack of flexible and high-performance proposal refinement remains a great hurdle for existing state-of-the-art two-stage detectors. Previous works on refining 3D proposals have relied on human-designed components such as keypoints sampling, set abstraction and multi-scale feature fusion to produce powerful 3D object representations. Such methods, however, have limited ability to capture rich contextual dependencies among points. In this paper, we leverage the high-quality region proposal network and a Channel-wise Transformer architecture to constitute our two-stage 3D object detection framework (CT3D) with minimal hand-crafted design. The proposed CT3D simultaneously performs proposal-aware embedding and channel-wise context aggregation for the point features within each proposal. Specifically, CT3D uses proposal's keypoints for spatial contextual modelling and learns attention propagation in the encoding module, mapping the proposal to point embeddings. Next, a new channel-wise decoding module enriches the query-key interaction via channel-wise re-weighting to effectively merge multi-level contexts, which contributes to more accurate object predictions. Extensive experiments demonstrate that our CT3D method has superior performance and excellent scalability. Remarkably, CT3D achieves the AP of 81.77% in the moderate car category on the KITTI test 3D detection benchmark, outperforms state-of-the-art 3D detectors.},
 bibtype = {article},
 author = {Sheng, Hualian and Cai, Sijia and Liu, Yuan and Deng, Bing and Huang, Jianqiang and Hua, Xian-Sheng and Zhao, Min-Jian}
}

Though 3D object detection from point clouds has achieved rapid progress in recent years, the lack of flexible and high-performance proposal refinement remains a great hurdle for existing state-of-the-art two-stage detectors. Previous works on refining 3D proposals have relied on human-designed components such as keypoints sampling, set abstraction and multi-scale feature fusion to produce powerful 3D object representations. Such methods, however, have limited ability to capture rich contextual dependencies among points. In this paper, we leverage the high-quality region proposal network and a Channel-wise Transformer architecture to constitute our two-stage 3D object detection framework (CT3D) with minimal hand-crafted design. The proposed CT3D simultaneously performs proposal-aware embedding and channel-wise context aggregation for the point features within each proposal. Specifically, CT3D uses proposal's keypoints for spatial contextual modelling and learns attention propagation in the encoding module, mapping the proposal to point embeddings. Next, a new channel-wise decoding module enriches the query-key interaction via channel-wise re-weighting to effectively merge multi-level contexts, which contributes to more accurate object predictions. Extensive experiments demonstrate that our CT3D method has superior performance and excellent scalability. Remarkably, CT3D achieves the AP of 81.77% in the moderate car category on the KITTI test 3D detection benchmark, outperforms state-of-the-art 3D detectors.

@article{
 title = {SAT: 2D Semantics Assisted Training for 3D Visual Grounding},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.11450},
 id = {96f6b47b-b381-334f-b899-cbf0bdc563f9},
 created = {2021-10-13T14:40:12.268Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.593Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {3D visual grounding aims at grounding a natural language description about a 3D scene, usually represented in the form of 3D point clouds, to the targeted object region. Point clouds are sparse, noisy, and contain limited semantic information compared with 2D images. These inherent limitations make the 3D visual grounding problem more challenging. In this study, we propose 2D Semantics Assisted Training (SAT) that utilizes 2D image semantics in the training stage to ease point-cloud-language joint representation learning and assist 3D visual grounding. The main idea is to learn auxiliary alignments between rich, clean 2D object representations and the corresponding objects or mentioned entities in 3D scenes. SAT takes 2D object semantics, i.e., object label, image feature, and 2D geometric feature, as the extra input in training but does not require such inputs during inference. By effectively utilizing 2D semantics in training, our approach boosts the accuracy on the Nr3D dataset from 37.7% to 49.2%, which significantly surpasses the non-SAT baseline with the identical network architecture and inference input. Our approach outperforms the state of the art by large margins on multiple 3D visual grounding datasets, i.e., +10.4% absolute accuracy on Nr3D, +9.9% on Sr3D, and +5.6% on ScanRef.},
 bibtype = {article},
 author = {Yang, Zhengyuan and Zhang, Songyang and Wang, Liwei and Luo, Jiebo}
}

3D visual grounding aims at grounding a natural language description about a 3D scene, usually represented in the form of 3D point clouds, to the targeted object region. Point clouds are sparse, noisy, and contain limited semantic information compared with 2D images. These inherent limitations make the 3D visual grounding problem more challenging. In this study, we propose 2D Semantics Assisted Training (SAT) that utilizes 2D image semantics in the training stage to ease point-cloud-language joint representation learning and assist 3D visual grounding. The main idea is to learn auxiliary alignments between rich, clean 2D object representations and the corresponding objects or mentioned entities in 3D scenes. SAT takes 2D object semantics, i.e., object label, image feature, and 2D geometric feature, as the extra input in training but does not require such inputs during inference. By effectively utilizing 2D semantics in training, our approach boosts the accuracy on the Nr3D dataset from 37.7% to 49.2%, which significantly surpasses the non-SAT baseline with the identical network architecture and inference input. Our approach outperforms the state of the art by large margins on multiple 3D visual grounding datasets, i.e., +10.4% absolute accuracy on Nr3D, +9.9% on Sr3D, and +5.6% on ScanRef.

@article{
 title = {GraphFPN: Graph Feature Pyramid Network for Object Detection},
 type = {article},
 year = {2021},
 pages = {2763-2772},
 websites = {http://arxiv.org/abs/2108.00580},
 id = {02650f40-5e6a-3fe8-95ed-c4036be55dc0},
 created = {2021-10-13T14:40:12.300Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-21T12:52:20.634Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Feature pyramids have been proven powerful in image understanding tasks that require multi-scale features. State-of-the-art methods for multi-scale feature learning focus on performing feature interactions across space and scales using neural networks with a fixed topology. In this paper, we propose graph feature pyramid networks that are capable of adapting their topological structures to varying intrinsic image structures and supporting simultaneous feature interactions across all scales. We first define an image-specific superpixel hierarchy for each input image to represent its intrinsic image structures. The graph feature pyramid network inherits its structure from this superpixel hierarchy. Contextual and hierarchical layers are designed to achieve feature interactions within the same scale and across different scales. To make these layers more powerful, we introduce two types of local channel attention for graph neural networks by generalizing global channel attention for convolutional neural networks. The proposed graph feature pyramid network can enhance the multiscale features from a convolutional feature pyramid network. We evaluate our graph feature pyramid network in the object detection task by integrating it into the Faster R-CNN algorithm. The modified algorithm outperforms not only previous state-of-the-art feature pyramid-based methods with a clear margin but also other popular detection methods on both MS-COCO 2017 validation and test datasets.},
 bibtype = {article},
 author = {Zhao, Gangming and Ge, Weifeng and Yu, Yizhou}
}

Feature pyramids have been proven powerful in image understanding tasks that require multi-scale features. State-of-the-art methods for multi-scale feature learning focus on performing feature interactions across space and scales using neural networks with a fixed topology. In this paper, we propose graph feature pyramid networks that are capable of adapting their topological structures to varying intrinsic image structures and supporting simultaneous feature interactions across all scales. We first define an image-specific superpixel hierarchy for each input image to represent its intrinsic image structures. The graph feature pyramid network inherits its structure from this superpixel hierarchy. Contextual and hierarchical layers are designed to achieve feature interactions within the same scale and across different scales. To make these layers more powerful, we introduce two types of local channel attention for graph neural networks by generalizing global channel attention for convolutional neural networks. The proposed graph feature pyramid network can enhance the multiscale features from a convolutional feature pyramid network. We evaluate our graph feature pyramid network in the object detection task by integrating it into the Faster R-CNN algorithm. The modified algorithm outperforms not only previous state-of-the-art feature pyramid-based methods with a clear margin but also other popular detection methods on both MS-COCO 2017 validation and test datasets.

@article{
 title = {Pyramid R-CNN: Towards Better Performance and Adaptability for 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {2723-2732},
 websites = {http://arxiv.org/abs/2109.02499},
 id = {78be5e7d-939c-3e1d-b52f-8dcc1101b692},
 created = {2021-10-13T14:40:12.310Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.362Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present a flexible and high-performance framework, named Pyramid R-CNN, for two-stage 3D object detection from point clouds. Current approaches generally rely on the points or voxels of interest for RoI feature extraction on the second stage, but cannot effectively handle the sparsity and non-uniform distribution of those points, and this may result in failures in detecting objects that are far away. To resolve the problems, we propose a novel second-stage module, named pyramid RoI head, to adaptively learn the features from the sparse points of interest. The pyramid RoI head consists of three key components. Firstly, we propose the RoI-grid Pyramid, which mitigates the sparsity problem by extensively collecting points of interest for each RoI in a pyramid manner. Secondly, we propose RoI-grid Attention, a new operation that can encode richer information from sparse points by incorporating conventional attention-based and graph-based point operators into a unified formulation. Thirdly, we propose the Density-Aware Radius Prediction (DARP) module, which can adapt to different point density levels by dynamically adjusting the focusing range of RoIs. Combining the three components, our pyramid RoI head is robust to the sparse and imbalanced circumstances, and can be applied upon various 3D backbones to consistently boost the detection performance. Extensive experiments show that Pyramid R-CNN outperforms the state-of-the-art 3D detection models by a large margin on both the KITTI dataset and the Waymo Open dataset.},
 bibtype = {article},
 author = {Mao, Jiageng and Niu, Minzhe and Bai, Haoyue and Liang, Xiaodan and Xu, Hang and Xu, Chunjing}
}

We present a flexible and high-performance framework, named Pyramid R-CNN, for two-stage 3D object detection from point clouds. Current approaches generally rely on the points or voxels of interest for RoI feature extraction on the second stage, but cannot effectively handle the sparsity and non-uniform distribution of those points, and this may result in failures in detecting objects that are far away. To resolve the problems, we propose a novel second-stage module, named pyramid RoI head, to adaptively learn the features from the sparse points of interest. The pyramid RoI head consists of three key components. Firstly, we propose the RoI-grid Pyramid, which mitigates the sparsity problem by extensively collecting points of interest for each RoI in a pyramid manner. Secondly, we propose RoI-grid Attention, a new operation that can encode richer information from sparse points by incorporating conventional attention-based and graph-based point operators into a unified formulation. Thirdly, we propose the Density-Aware Radius Prediction (DARP) module, which can adapt to different point density levels by dynamically adjusting the focusing range of RoIs. Combining the three components, our pyramid RoI head is robust to the sparse and imbalanced circumstances, and can be applied upon various 3D backbones to consistently boost the detection performance. Extensive experiments show that Pyramid R-CNN outperforms the state-of-the-art 3D detection models by a large margin on both the KITTI dataset and the Waymo Open dataset.

@article{
 title = {Graph Constrained Data Representation Learning for Human Motion Segmentation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.13362},
 id = {370c78a5-e88b-340b-af27-c92fe7dbe51d},
 created = {2021-10-13T14:40:12.326Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.937Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Recently, transfer subspace learning based approaches have shown to be a valid alternative to unsupervised subspace clustering and temporal data clustering for human motion segmentation (HMS). These approaches leverage prior knowledge from a source domain to improve clustering performance on a target domain, and currently they represent the state of the art in HMS. Bucking this trend, in this paper, we propose a novel unsupervised model that learns a representation of the data and digs clustering information from the data itself. Our model is reminiscent of temporal subspace clustering, but presents two critical differences. First, we learn an auxiliary data matrix that can deviate from the initial data, hence confer more degrees of freedom to the coding matrix. Second, we introduce a regularization term for this auxiliary data matrix that preserves the local geometrical structure present in the high-dimensional space. The proposed model is efficiently optimized by using an original Alternating Direction Method of Multipliers (ADMM) formulation allowing to learn jointly the auxiliary data representation, a nonnegative dictionary and a coding matrix. Experimental results on four benchmark datasets for HMS demonstrate that our approach achieves significantly better clustering performance then state-of-the-art methods, including both unsupervised and more recent semi-supervised transfer learning approaches.},
 bibtype = {article},
 author = {Dimiccoli, Mariella and Garrido, Lluís and Rodriguez-Corominas, Guillem and Wendt, Herwig}
}

Recently, transfer subspace learning based approaches have shown to be a valid alternative to unsupervised subspace clustering and temporal data clustering for human motion segmentation (HMS). These approaches leverage prior knowledge from a source domain to improve clustering performance on a target domain, and currently they represent the state of the art in HMS. Bucking this trend, in this paper, we propose a novel unsupervised model that learns a representation of the data and digs clustering information from the data itself. Our model is reminiscent of temporal subspace clustering, but presents two critical differences. First, we learn an auxiliary data matrix that can deviate from the initial data, hence confer more degrees of freedom to the coding matrix. Second, we introduce a regularization term for this auxiliary data matrix that preserves the local geometrical structure present in the high-dimensional space. The proposed model is efficiently optimized by using an original Alternating Direction Method of Multipliers (ADMM) formulation allowing to learn jointly the auxiliary data representation, a nonnegative dictionary and a coding matrix. Experimental results on four benchmark datasets for HMS demonstrate that our approach achieves significantly better clustering performance then state-of-the-art methods, including both unsupervised and more recent semi-supervised transfer learning approaches.

@article{
 title = {Deep 3D Mask Volume for View Synthesis of Dynamic Scenes},
 type = {article},
 year = {2021},
 pages = {1749-1758},
 websites = {http://cseweb.ucsd.edu/},
 id = {d2183210-bb06-3850-ac1e-4d65fcc8e9a2},
 created = {2021-10-13T14:40:12.354Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:29.765Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Image view synthesis has seen great success in reconstructing photorealistic visuals, thanks to deep learning and various novel representations. The next key step in immer-sive virtual experiences is view synthesis of dynamic scenes. However, several challenges exist due to the lack of high-quality training datasets, and the additional time dimension for videos of dynamic scenes. To address this issue, we introduce a multi-view video dataset, captured with a custom 10-camera rig in 120FPS. The dataset contains 96 high-quality scenes showing various visual effects and human interactions in outdoor scenes. We develop a new algorithm , Deep 3D Mask Volume, which enables temporally-stable view extrapolation from binocular videos of dynamic scenes, captured by static cameras. Our algorithm addresses the temporal inconsistency of disocclusions by identifying the error-prone areas with a 3D mask volume, and replaces them with static background observed throughout the video. Our method enables manipulation in 3D space as opposed to simple 2D masks, We demonstrate better temporal stability than frame-by-frame static view synthesis methods , or those that use 2D masks. The resulting view synthesis videos show minimal flickering artifacts and allow for larger translational movements.},
 bibtype = {article},
 author = {Lin, Kai-En and Xiao, Lei and Liu, Feng and Yang, Guowei and Ramamoorthi, Ravi},
 journal = {Iccv}
}

Image view synthesis has seen great success in reconstructing photorealistic visuals, thanks to deep learning and various novel representations. The next key step in immer-sive virtual experiences is view synthesis of dynamic scenes. However, several challenges exist due to the lack of high-quality training datasets, and the additional time dimension for videos of dynamic scenes. To address this issue, we introduce a multi-view video dataset, captured with a custom 10-camera rig in 120FPS. The dataset contains 96 high-quality scenes showing various visual effects and human interactions in outdoor scenes. We develop a new algorithm , Deep 3D Mask Volume, which enables temporally-stable view extrapolation from binocular videos of dynamic scenes, captured by static cameras. Our algorithm addresses the temporal inconsistency of disocclusions by identifying the error-prone areas with a 3D mask volume, and replaces them with static background observed throughout the video. Our method enables manipulation in 3D space as opposed to simple 2D masks, We demonstrate better temporal stability than frame-by-frame static view synthesis methods , or those that use 2D masks. The resulting view synthesis videos show minimal flickering artifacts and allow for larger translational movements.

@article{
 title = {Learning Canonical 3D Object Representation for Fine-Grained Recognition},
 type = {article},
 year = {2021},
 pages = {1035-1045},
 websites = {http://arxiv.org/abs/2108.04628},
 id = {393d4a0b-b7d8-3856-b0c7-a110ccbe53c2},
 created = {2021-10-13T14:40:12.460Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:30.413Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We propose a novel framework for fine-grained object recognition that learns to recover object variation in 3D space from a single image, trained on an image collection without using any ground-truth 3D annotation. We accomplish this by representing an object as a composition of 3D shape and its appearance, while eliminating the effect of camera viewpoint, in a canonical configuration. Unlike conventional methods modeling spatial variation in 2D images only, our method is capable of reconfiguring the appearance feature in a canonical 3D space, thus enabling the subsequent object classifier to be invariant under 3D geometric variation. Our representation also allows us to go beyond existing methods, by incorporating 3D shape variation as an additional cue for object recognition. To learn the model without ground-truth 3D annotation, we deploy a differentiable renderer in an analysis-by-synthesis framework. By incorporating 3D shape and appearance jointly in a deep representation, our method learns the discriminative representation of the object and achieves competitive performance on fine-grained image recognition and vehicle re-identification. We also demonstrate that the performance of 3D shape reconstruction is improved by learning fine-grained shape deformation in a boosting manner.},
 bibtype = {article},
 author = {Joung, Sunghun and Kim, Seungryong and Kim, Minsu and Kim, Ig-Jae and Sohn, Kwanghoon},
 number = {c}
}

We propose a novel framework for fine-grained object recognition that learns to recover object variation in 3D space from a single image, trained on an image collection without using any ground-truth 3D annotation. We accomplish this by representing an object as a composition of 3D shape and its appearance, while eliminating the effect of camera viewpoint, in a canonical configuration. Unlike conventional methods modeling spatial variation in 2D images only, our method is capable of reconfiguring the appearance feature in a canonical 3D space, thus enabling the subsequent object classifier to be invariant under 3D geometric variation. Our representation also allows us to go beyond existing methods, by incorporating 3D shape variation as an additional cue for object recognition. To learn the model without ground-truth 3D annotation, we deploy a differentiable renderer in an analysis-by-synthesis framework. By incorporating 3D shape and appearance jointly in a deep representation, our method learns the discriminative representation of the object and achieves competitive performance on fine-grained image recognition and vehicle re-identification. We also demonstrate that the performance of 3D shape reconstruction is improved by learning fine-grained shape deformation in a boosting manner.

@article{
 title = {PICCOLO: Point Cloud-Centric Omnidirectional Localization},
 type = {article},
 year = {2021},
 pages = {3313-3323},
 websites = {http://arxiv.org/abs/2108.06545},
 id = {12c4d492-a378-35f2-992f-129a41704319},
 created = {2021-10-13T14:40:12.556Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:26.341Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present PICCOLO, a simple and efficient algorithm for omnidirectional localization. Given a colored point cloud and a 360 panorama image of a scene, our objective is to recover the camera pose at which the panorama image is taken. Our pipeline works in an off-the-shelf manner with a single image given as a query and does not require any training of neural networks or collecting ground-truth poses of images. Instead, we match each point cloud color to the holistic view of the panorama image with gradient-descent optimization to find the camera pose. Our loss function, called sampling loss, is point cloud-centric, evaluated at the projected location of every point in the point cloud. In contrast, conventional photometric loss is image-centric, comparing colors at each pixel location. With a simple change in the compared entities, sampling loss effectively overcomes the severe visual distortion of omnidirectional images, and enjoys the global context of the 360 view to handle challenging scenarios for visual localization. PICCOLO outperforms existing omnidirectional localization algorithms in both accuracy and stability when evaluated in various environments.},
 bibtype = {article},
 author = {Kim, Junho and Choi, Changwoon and Jang, Hojun and Kim, Young Min}
}

We present PICCOLO, a simple and efficient algorithm for omnidirectional localization. Given a colored point cloud and a 360 panorama image of a scene, our objective is to recover the camera pose at which the panorama image is taken. Our pipeline works in an off-the-shelf manner with a single image given as a query and does not require any training of neural networks or collecting ground-truth poses of images. Instead, we match each point cloud color to the holistic view of the panorama image with gradient-descent optimization to find the camera pose. Our loss function, called sampling loss, is point cloud-centric, evaluated at the projected location of every point in the point cloud. In contrast, conventional photometric loss is image-centric, comparing colors at each pixel location. With a simple change in the compared entities, sampling loss effectively overcomes the severe visual distortion of omnidirectional images, and enjoys the global context of the 360 view to handle challenging scenarios for visual localization. PICCOLO outperforms existing omnidirectional localization algorithms in both accuracy and stability when evaluated in various environments.

@article{
 title = {Exploring Geometry-aware Contrast and Clustering Harmonization for Self-supervised 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {3293-3302},
 id = {bd253c61-4863-3a96-a097-ef8ca287a263},
 created = {2021-10-13T14:40:12.588Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:26.674Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Current 3D object detection paradigms highly rely on extensive annotation efforts, which makes them not practical in many real-world industrial applications. Inspired by that a human driver can keep accumulating experiences from self-exploring the roads without any tutor's guidance, we first step forwards to explore a simple yet effective self-supervised learning framework tailored for LiDAR-based 3D object detection. Although the self-supervised pipeline has achieved great success in 2D domain, the characteristic challenges (e.g., complex geometry structure and various 3D object views) encountered in the 3D domain hinder the direct adoption of existing techniques that often contrast the 2D augmented data or cluster single-view features. Here we present a novel self-supervised 3D Object detection framework that seamlessly integrates the geometry-aware contrast and clustering harmonization to lift the unsupervised 3D representation learning, named GCC-3D. First, GCC-3D introduces a Geometric-Aware Contrastive objective to learn spatial-sensitive local structure representation. This objective enforces the spatially close voxels to have high feature similarity. Second, a Pseudo-Instance Clustering harmonization mechanism is proposed to encourage that different views of pseudo-instances should have consistent similarities to clustering prototype centers. This module endows our model semantic discriminative capacity. Extensive experiments demonstrate our GCC-3D achieves significant performance improvement on data-efficient 3D object detection benchmarks (nuScenes and Waymo). Moreover, our GCC-3D framework can achieve state-of-the art performances on all popular 3D object detection benchmarks.},
 bibtype = {article},
 author = {Iccv, Anonymous and Id, Paper}
}

Current 3D object detection paradigms highly rely on extensive annotation efforts, which makes them not practical in many real-world industrial applications. Inspired by that a human driver can keep accumulating experiences from self-exploring the roads without any tutor's guidance, we first step forwards to explore a simple yet effective self-supervised learning framework tailored for LiDAR-based 3D object detection. Although the self-supervised pipeline has achieved great success in 2D domain, the characteristic challenges (e.g., complex geometry structure and various 3D object views) encountered in the 3D domain hinder the direct adoption of existing techniques that often contrast the 2D augmented data or cluster single-view features. Here we present a novel self-supervised 3D Object detection framework that seamlessly integrates the geometry-aware contrast and clustering harmonization to lift the unsupervised 3D representation learning, named GCC-3D. First, GCC-3D introduces a Geometric-Aware Contrastive objective to learn spatial-sensitive local structure representation. This objective enforces the spatially close voxels to have high feature similarity. Second, a Pseudo-Instance Clustering harmonization mechanism is proposed to encourage that different views of pseudo-instances should have consistent similarities to clustering prototype centers. This module endows our model semantic discriminative capacity. Extensive experiments demonstrate our GCC-3D achieves significant performance improvement on data-efficient 3D object detection benchmarks (nuScenes and Waymo). Moreover, our GCC-3D framework can achieve state-of-the art performances on all popular 3D object detection benchmarks.

@article{
 title = {Long-Term Temporally Consistent Unpaired Video Translation from Simulated Surgical 3D Data},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.17204},
 id = {cc736110-7878-36e1-9c8e-ecfa98d2aa38},
 created = {2021-10-13T14:40:12.590Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:26.164Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Research in unpaired video translation has mainly focused on short-term temporal consistency by conditioning on neighboring frames. However for transfer from simulated to photorealistic sequences, available information on the underlying geometry offers potential for achieving global consistency across views. We propose a novel approach which combines unpaired image translation with neural rendering to transfer simulated to photorealistic surgical abdominal scenes. By introducing global learnable textures and a lighting-invariant view-consistency loss, our method produces consistent translations of arbitrary views and thus enables long-term consistent video synthesis. We design and test our model to generate video sequences from minimally-invasive surgical abdominal scenes. Because labeled data is often limited in this domain, photorealistic data where ground truth information from the simulated domain is preserved is especially relevant. By extending existing image-based methods to view-consistent videos, we aim to impact the applicability of simulated training and evaluation environments for surgical applications. Code and data will be made publicly available soon.},
 bibtype = {article},
 author = {Rivoir, Dominik and Pfeiffer, Micha and Docea, Reuben and Kolbinger, Fiona and Riediger, Carina and Weitz, Jürgen and Speidel, Stefanie}
}

Research in unpaired video translation has mainly focused on short-term temporal consistency by conditioning on neighboring frames. However for transfer from simulated to photorealistic sequences, available information on the underlying geometry offers potential for achieving global consistency across views. We propose a novel approach which combines unpaired image translation with neural rendering to transfer simulated to photorealistic surgical abdominal scenes. By introducing global learnable textures and a lighting-invariant view-consistency loss, our method produces consistent translations of arbitrary views and thus enables long-term consistent video synthesis. We design and test our model to generate video sequences from minimally-invasive surgical abdominal scenes. Because labeled data is often limited in this domain, photorealistic data where ground truth information from the simulated domain is preserved is especially relevant. By extending existing image-based methods to view-consistent videos, we aim to impact the applicability of simulated training and evaluation environments for surgical applications. Code and data will be made publicly available soon.

@article{
 title = {RePOSE: Fast 6D Object Pose Refinement via Deep Texture Rendering},
 type = {article},
 year = {2021},
 pages = {3303-3312},
 websites = {http://arxiv.org/abs/2104.00633},
 id = {7b0a1c13-c317-3807-9f21-a1c1585c625a},
 created = {2021-10-13T14:40:12.643Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:26.517Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present RePOSE, a fast iterative refinement method for 6D object pose estimation. Prior methods perform refinement by feeding zoomed-in input and rendered RGB images into a CNN and directly regressing an update of a refined pose. Their runtime is slow due to the computational cost of CNN, which is especially prominent in multiple-object pose refinement. To overcome this problem, RePOSE leverages image rendering for fast feature extraction using a 3D model with a learnable texture. We call this deep texture rendering, which uses a shallow multi-layer perceptron to directly regress a view-invariant image representation of an object. Furthermore, we utilize differentiable Levenberg-Marquardt (LM) optimization to refine a pose fast and accurately by minimizing the feature-metric error between the input and rendered image representations without the need of zooming in. These image representations are trained such that differentiable LM optimization converges within few iterations. Consequently, RePOSE runs at 92 FPS and achieves state-of-the-art accuracy of 51.6% on the Occlusion LineMOD dataset - a 4.1% absolute improvement over the prior art, and comparable result on the YCB-Video dataset with a much faster runtime. The code is available at https://github.com/sh8/repose.},
 bibtype = {article},
 author = {Iwase, Shun and Liu, Xingyu and Khirodkar, Rawal and Yokota, Rio and Kitani, Kris M.}
}

We present RePOSE, a fast iterative refinement method for 6D object pose estimation. Prior methods perform refinement by feeding zoomed-in input and rendered RGB images into a CNN and directly regressing an update of a refined pose. Their runtime is slow due to the computational cost of CNN, which is especially prominent in multiple-object pose refinement. To overcome this problem, RePOSE leverages image rendering for fast feature extraction using a 3D model with a learnable texture. We call this deep texture rendering, which uses a shallow multi-layer perceptron to directly regress a view-invariant image representation of an object. Furthermore, we utilize differentiable Levenberg-Marquardt (LM) optimization to refine a pose fast and accurately by minimizing the feature-metric error between the input and rendered image representations without the need of zooming in. These image representations are trained such that differentiable LM optimization converges within few iterations. Consequently, RePOSE runs at 92 FPS and achieves state-of-the-art accuracy of 51.6% on the Occlusion LineMOD dataset - a 4.1% absolute improvement over the prior art, and comparable result on the YCB-Video dataset with a much faster runtime. The code is available at https://github.com/sh8/repose.

@article{
 title = {Voxel Transformer for 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {3164-3173},
 websites = {http://arxiv.org/abs/2109.02497},
 id = {c78ba4a1-2aed-3161-8a95-563af37e1f4f},
 created = {2021-10-13T14:40:12.667Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:26.982Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present Voxel Transformer (VoTr), a novel and effective voxel-based Transformer backbone for 3D object detection from point clouds. Conventional 3D convolutional backbones in voxel-based 3D detectors cannot efficiently capture large context information, which is crucial for object recognition and localization, owing to the limited receptive fields. In this paper, we resolve the problem by introducing a Transformer-based architecture that enables long-range relationships between voxels by self-attention. Given the fact that non-empty voxels are naturally sparse but numerous, directly applying standard Transformer on voxels is non-trivial. To this end, we propose the sparse voxel module and the submanifold voxel module, which can operate on the empty and non-empty voxel positions effectively. To further enlarge the attention range while maintaining comparable computational overhead to the convolutional counterparts, we propose two attention mechanisms for multi-head attention in those two modules: Local Attention and Dilated Attention, and we further propose Fast Voxel Query to accelerate the querying process in multi-head attention. VoTr contains a series of sparse and submanifold voxel modules and can be applied in most voxel-based detectors. Our proposed VoTr shows consistent improvement over the convolutional baselines while maintaining computational efficiency on the KITTI dataset and the Waymo Open dataset.},
 bibtype = {article},
 author = {Mao, Jiageng and Xue, Yujing and Niu, Minzhe and Bai, Haoyue and Feng, Jiashi and Liang, Xiaodan and Xu, Hang and Xu, Chunjing}
}

We present Voxel Transformer (VoTr), a novel and effective voxel-based Transformer backbone for 3D object detection from point clouds. Conventional 3D convolutional backbones in voxel-based 3D detectors cannot efficiently capture large context information, which is crucial for object recognition and localization, owing to the limited receptive fields. In this paper, we resolve the problem by introducing a Transformer-based architecture that enables long-range relationships between voxels by self-attention. Given the fact that non-empty voxels are naturally sparse but numerous, directly applying standard Transformer on voxels is non-trivial. To this end, we propose the sparse voxel module and the submanifold voxel module, which can operate on the empty and non-empty voxel positions effectively. To further enlarge the attention range while maintaining comparable computational overhead to the convolutional counterparts, we propose two attention mechanisms for multi-head attention in those two modules: Local Attention and Dilated Attention, and we further propose Fast Voxel Query to accelerate the querying process in multi-head attention. VoTr contains a series of sparse and submanifold voxel modules and can be applied in most voxel-based detectors. Our proposed VoTr shows consistent improvement over the convolutional baselines while maintaining computational efficiency on the KITTI dataset and the Waymo Open dataset.

@article{
 title = {Geometry Uncertainty Projection Network for Monocular 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {3111-3121},
 websites = {http://arxiv.org/abs/2107.13774},
 id = {b540c9cc-0c72-317d-ac8f-92bd8583ece1},
 created = {2021-10-13T14:40:12.707Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:05.732Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Geometry Projection is a powerful depth estimation method in monocular 3D object detection. It estimates depth dependent on heights, which introduces mathematical priors into the deep model. But projection process also introduces the error amplification problem, in which the error of the estimated height will be amplified and reflected greatly at the output depth. This property leads to uncontrollable depth inferences and also damages the training efficiency. In this paper, we propose a Geometry Uncertainty Projection Network (GUP Net) to tackle the error amplification problem at both inference and training stages. Specifically, a GUP module is proposed to obtains the geometry-guided uncertainty of the inferred depth, which not only provides high reliable confidence for each depth but also benefits depth learning. Furthermore, at the training stage, we propose a Hierarchical Task Learning strategy to reduce the instability caused by error amplification. This learning algorithm monitors the learning situation of each task by a proposed indicator and adaptively assigns the proper loss weights for different tasks according to their pre-tasks situation. Based on that, each task starts learning only when its pre-tasks are learned well, which can significantly improve the stability and efficiency of the training process. Extensive experiments demonstrate the effectiveness of the proposed method. The overall model can infer more reliable object depth than existing methods and outperforms the state-of-the-art image-based monocular 3D detectors by 3.74% and 4.7% AP40 of the car and pedestrian categories on the KITTI benchmark.},
 bibtype = {article},
 author = {Lu, Yan and Ma, Xinzhu and Yang, Lei and Zhang, Tianzhu and Liu, Yating and Chu, Qi and Yan, Junjie and Ouyang, Wanli}
}

Geometry Projection is a powerful depth estimation method in monocular 3D object detection. It estimates depth dependent on heights, which introduces mathematical priors into the deep model. But projection process also introduces the error amplification problem, in which the error of the estimated height will be amplified and reflected greatly at the output depth. This property leads to uncontrollable depth inferences and also damages the training efficiency. In this paper, we propose a Geometry Uncertainty Projection Network (GUP Net) to tackle the error amplification problem at both inference and training stages. Specifically, a GUP module is proposed to obtains the geometry-guided uncertainty of the inferred depth, which not only provides high reliable confidence for each depth but also benefits depth learning. Furthermore, at the training stage, we propose a Hierarchical Task Learning strategy to reduce the instability caused by error amplification. This learning algorithm monitors the learning situation of each task by a proposed indicator and adaptively assigns the proper loss weights for different tasks according to their pre-tasks situation. Based on that, each task starts learning only when its pre-tasks are learned well, which can significantly improve the stability and efficiency of the training process. Extensive experiments demonstrate the effectiveness of the proposed method. The overall model can infer more reliable object depth than existing methods and outperforms the state-of-the-art image-based monocular 3D detectors by 3.74% and 4.7% AP40 of the car and pedestrian categories on the KITTI benchmark.

@article{
 title = {Group-Free 3D Object Detection via Transformers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.00678},
 id = {07c4dd4d-e090-3886-884c-dcb94cfbae63},
 created = {2021-10-13T14:40:12.714Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:27.687Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b,be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {Recently, directly detecting 3D objects from 3D point clouds has received increasing attention. To extract object representation from an irregular point cloud, existing methods usually take a point grouping step to assign the points to an object candidate so that a PointNet-like network could be used to derive object features from the grouped points. However, the inaccurate point assignments caused by the hand-crafted grouping scheme decrease the performance of 3D object detection. In this paper, we present a simple yet effective method for directly detecting 3D objects from the 3D point cloud. Instead of grouping local points to each object candidate, our method computes the feature of an object from all the points in the point cloud with the help of an attention mechanism in the Transformers \citevaswani2017attention, where the contribution of each point is automatically learned in the network training. With an improved attention stacking scheme, our method fuses object features in different stages and generates more accurate object detection results. With few bells and whistles, the proposed method achieves state-of-the-art 3D object detection performance on two widely used benchmarks, ScanNet V2 and SUN RGB-D. The code and models are publicly available at \urlhttps://github.com/zeliu98/Group-Free-3D},
 bibtype = {article},
 author = {Liu, Ze and Zhang, Zheng and Cao, Yue and Hu, Han and Tong, Xin}
}

Recently, directly detecting 3D objects from 3D point clouds has received increasing attention. To extract object representation from an irregular point cloud, existing methods usually take a point grouping step to assign the points to an object candidate so that a PointNet-like network could be used to derive object features from the grouped points. However, the inaccurate point assignments caused by the hand-crafted grouping scheme decrease the performance of 3D object detection. In this paper, we present a simple yet effective method for directly detecting 3D objects from the 3D point cloud. Instead of grouping local points to each object candidate, our method computes the feature of an object from all the points in the point cloud with the help of an attention mechanism in the Transformers \citevaswani2017attention, where the contribution of each point is automatically learned in the network training. With an improved attention stacking scheme, our method fuses object features in different stages and generates more accurate object detection results. With few bells and whistles, the proposed method achieves state-of-the-art 3D object detection performance on two widely used benchmarks, ScanNet V2 and SUN RGB-D. The code and models are publicly available at \urlhttps://github.com/zeliu98/Group-Free-3D

@article{
 title = {End-to-End Semi-Supervised Object Detection with Soft Teacher},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2106.09018},
 id = {1d72ff15-5bc1-3f73-b62d-22a1137fc0c9},
 created = {2021-10-13T14:40:12.773Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:27.476Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {This paper presents an end-to-end semi-supervised object detection approach, in contrast to previous more complex multi-stage methods. The end-to-end training gradually improves pseudo label qualities during the curriculum, and the more and more accurate pseudo labels in turn benefit object detection training. We also propose two simple yet effective techniques within this framework: a soft teacher mechanism where the classification loss of each unlabeled bounding box is weighed by the classification score produced by the teacher network; a box jittering approach to select reliable pseudo boxes for the learning of box regression. On the COCO benchmark, the proposed approach outperforms previous methods by a large margin under various labeling ratios, i.e. 1\%, 5\% and 10\%. Moreover, our approach proves to perform also well when the amount of labeled data is relatively large. For example, it can improve a 40.9 mAP baseline detector trained using the full COCO training set by +3.6 mAP, reaching 44.5 mAP, by leveraging the 123K unlabeled images of COCO. On the state-of-the-art Swin Transformer based object detector (58.9 mAP on test-dev), it can still significantly improve the detection accuracy by +1.5 mAP, reaching 60.4 mAP, and improve the instance segmentation accuracy by +1.2 mAP, reaching 52.4 mAP. Further incorporating with the Object365 pre-trained model, the detection accuracy reaches 61.3 mAP and the instance segmentation accuracy reaches 53.0 mAP, pushing the new state-of-the-art.},
 bibtype = {article},
 author = {Xu, Mengde and Zhang, Zheng and Hu, Han and Wang, Jianfeng and Wang, Lijuan and Wei, Fangyun and Bai, Xiang and Liu, Zicheng}
}

This paper presents an end-to-end semi-supervised object detection approach, in contrast to previous more complex multi-stage methods. The end-to-end training gradually improves pseudo label qualities during the curriculum, and the more and more accurate pseudo labels in turn benefit object detection training. We also propose two simple yet effective techniques within this framework: a soft teacher mechanism where the classification loss of each unlabeled bounding box is weighed by the classification score produced by the teacher network; a box jittering approach to select reliable pseudo boxes for the learning of box regression. On the COCO benchmark, the proposed approach outperforms previous methods by a large margin under various labeling ratios, i.e. 1\%, 5\% and 10\%. Moreover, our approach proves to perform also well when the amount of labeled data is relatively large. For example, it can improve a 40.9 mAP baseline detector trained using the full COCO training set by +3.6 mAP, reaching 44.5 mAP, by leveraging the 123K unlabeled images of COCO. On the state-of-the-art Swin Transformer based object detector (58.9 mAP on test-dev), it can still significantly improve the detection accuracy by +1.5 mAP, reaching 60.4 mAP, and improve the instance segmentation accuracy by +1.2 mAP, reaching 52.4 mAP. Further incorporating with the Object365 pre-trained model, the detection accuracy reaches 61.3 mAP and the instance segmentation accuracy reaches 53.0 mAP, pushing the new state-of-the-art.

@article{
 title = {An End-to-End Transformer Model for 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {2906-2917},
 websites = {http://arxiv.org/abs/2109.08141},
 id = {f4a99d77-13b6-38a7-988d-7ff0628a36d4},
 created = {2021-10-13T14:40:12.784Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:27.985Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We propose 3DETR, an end-to-end Transformer based object detection model for 3D point clouds. Compared to existing detection methods that employ a number of 3D-specific inductive biases, 3DETR requires minimal modifications to the vanilla Transformer block. Specifically, we find that a standard Transformer with non-parametric queries and Fourier positional embeddings is competitive with specialized architectures that employ libraries of 3D-specific operators with hand-tuned hyperparameters. Nevertheless, 3DETR is conceptually simple and easy to implement, enabling further improvements by incorporating 3D domain knowledge. Through extensive experiments, we show 3DETR outperforms the well-established and highly optimized VoteNet baselines on the challenging ScanNetV2 dataset by 9.5%. Furthermore, we show 3DETR is applicable to 3D tasks beyond detection, and can serve as a building block for future research.},
 bibtype = {article},
 author = {Misra, Ishan and Girdhar, Rohit and Joulin, Armand}
}

We propose 3DETR, an end-to-end Transformer based object detection model for 3D point clouds. Compared to existing detection methods that employ a number of 3D-specific inductive biases, 3DETR requires minimal modifications to the vanilla Transformer block. Specifically, we find that a standard Transformer with non-parametric queries and Fourier positional embeddings is competitive with specialized architectures that employ libraries of 3D-specific operators with hand-tuned hyperparameters. Nevertheless, 3DETR is conceptually simple and easy to implement, enabling further improvements by incorporating 3D domain knowledge. Through extensive experiments, we show 3DETR outperforms the well-established and highly optimized VoteNet baselines on the challenging ScanNetV2 dataset by 9.5%. Furthermore, we show 3DETR is applicable to 3D tasks beyond detection, and can serve as a building block for future research.

@article{
 title = {MLVSNet : Multi-level Voting Siamese Network for 3D Visual Tracking},
 type = {article},
 year = {2021},
 id = {445b4dd8-97db-341c-a89e-a9da0522cf06},
 created = {2021-10-13T14:40:12.810Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:27.308Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {be408929-b86c-42e8-9ccd-02ff8d8707f0,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 bibtype = {article},
 author = {Area, Search}
}

@article{
 title = {CvT: Introducing Convolutions to Vision Transformers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.15808},
 month = {3},
 day = {29},
 id = {2c082ca1-733a-303b-bc39-2e9f30ce0f0c},
 created = {2021-10-14T06:53:42.895Z},
 accessed = {2021-10-14},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-18T06:16:31.966Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b,b09853a1-601b-4dfc-9176-c3c7d469342b},
 private_publication = {false},
 abstract = {We present in this paper a new architecture, named Convolutional vision Transformer (CvT), that improves Vision Transformer (ViT) in performance and efficiency by introducing convolutions into ViT to yield the best of both designs. This is accomplished through two primary modifications: a hierarchy of Transformers containing a new convolutional token embedding, and a convolutional Transformer block leveraging a convolutional projection. These changes introduce desirable properties of convolutional neural networks (CNNs) to the ViT architecture (\ie shift, scale, and distortion invariance) while maintaining the merits of Transformers (\ie dynamic attention, global context, and better generalization). We validate CvT by conducting extensive experiments, showing that this approach achieves state-of-the-art performance over other Vision Transformers and ResNets on ImageNet-1k, with fewer parameters and lower FLOPs. In addition, performance gains are maintained when pretrained on larger datasets (\eg ImageNet-22k) and fine-tuned to downstream tasks. Pre-trained on ImageNet-22k, our CvT-W24 obtains a top-1 accuracy of 87.7\% on the ImageNet-1k val set. Finally, our results show that the positional encoding, a crucial component in existing Vision Transformers, can be safely removed in our model, simplifying the design for higher resolution vision tasks. Code will be released at \urlhttps://github.com/leoxiaobin/CvT.},
 bibtype = {article},
 author = {Wu, Haiping and Xiao, Bin and Codella, Noel and Liu, Mengchen and Dai, Xiyang and Yuan, Lu and Zhang, Lei}
}

We present in this paper a new architecture, named Convolutional vision Transformer (CvT), that improves Vision Transformer (ViT) in performance and efficiency by introducing convolutions into ViT to yield the best of both designs. This is accomplished through two primary modifications: a hierarchy of Transformers containing a new convolutional token embedding, and a convolutional Transformer block leveraging a convolutional projection. These changes introduce desirable properties of convolutional neural networks (CNNs) to the ViT architecture (\ie shift, scale, and distortion invariance) while maintaining the merits of Transformers (\ie dynamic attention, global context, and better generalization). We validate CvT by conducting extensive experiments, showing that this approach achieves state-of-the-art performance over other Vision Transformers and ResNets on ImageNet-1k, with fewer parameters and lower FLOPs. In addition, performance gains are maintained when pretrained on larger datasets (\eg ImageNet-22k) and fine-tuned to downstream tasks. Pre-trained on ImageNet-22k, our CvT-W24 obtains a top-1 accuracy of 87.7\% on the ImageNet-1k val set. Finally, our results show that the positional encoding, a crucial component in existing Vision Transformers, can be safely removed in our model, simplifying the design for higher resolution vision tasks. Code will be released at \urlhttps://github.com/leoxiaobin/CvT.

@article{
 title = {CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification},
 type = {article},
 year = {2021},
 websites = {https://github.com/IBM/CrossViT.},
 id = {85185768-acea-33f0-b3ae-f71eb62d4cf7},
 created = {2021-10-14T06:57:45.679Z},
 accessed = {2021-10-14},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:17:05.839Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {The recently developed vision transformer (ViT) has achieved promising results on image classification compared to convolutional neural networks. Inspired by this, in this paper, we study how to learn multi-scale feature representations in transformer models for image classification. To this end, we propose a dual-branch transformer to combine image patches (i.e., tokens in a transformer) of different sizes to produce stronger image features. Our approach processes small-patch and large-patch tokens with two separate branches of different computational complexity and these tokens are then fused purely by attention multiple times to complement each other. Furthermore, to reduce computation, we develop a simple yet effective token fusion module based on cross attention, which uses a single token for each branch as a query to exchange information with other branches. Our proposed cross-attention only requires linear time for both computational and memory complexity instead of quadratic time otherwise. Extensive experiments demonstrate that our approach performs better than or on par with several concurrent works on vision transformer, in addition to efficient CNN models. For example, on the ImageNet1K dataset, with some architectural changes, our approach outperforms the recent DeiT by a large margin of 2% with a small to moderate increase in FLOPs and model parameters. Our source codes and models are available at https://github.com/IBM/CrossViT.},
 bibtype = {article},
 author = {Chen, Chun-Fu and Fan, Quanfu and Panda, Rameswar}
}

The recently developed vision transformer (ViT) has achieved promising results on image classification compared to convolutional neural networks. Inspired by this, in this paper, we study how to learn multi-scale feature representations in transformer models for image classification. To this end, we propose a dual-branch transformer to combine image patches (i.e., tokens in a transformer) of different sizes to produce stronger image features. Our approach processes small-patch and large-patch tokens with two separate branches of different computational complexity and these tokens are then fused purely by attention multiple times to complement each other. Furthermore, to reduce computation, we develop a simple yet effective token fusion module based on cross attention, which uses a single token for each branch as a query to exchange information with other branches. Our proposed cross-attention only requires linear time for both computational and memory complexity instead of quadratic time otherwise. Extensive experiments demonstrate that our approach performs better than or on par with several concurrent works on vision transformer, in addition to efficient CNN models. For example, on the ImageNet1K dataset, with some architectural changes, our approach outperforms the recent DeiT by a large margin of 2% with a small to moderate increase in FLOPs and model parameters. Our source codes and models are available at https://github.com/IBM/CrossViT.

@article{
 title = {Learning Spatio-Temporal Transformer for Visual Tracking},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.17154},
 id = {78ccd856-4cfa-3151-9e01-3d3f5c2c2ca3},
 created = {2021-10-14T07:07:44.819Z},
 file_attached = {false},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:09:54.771Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {In this paper, we present a new tracking architecture with an encoder-decoder transformer as the key component. The encoder models the global spatio-temporal feature dependencies between target objects and search regions, while the decoder learns a query embedding to predict the spatial positions of the target objects. Our method casts object tracking as a direct bounding box prediction problem, without using any proposals or predefined anchors. With the encoder-decoder transformer, the prediction of objects just uses a simple fully-convolutional network, which estimates the corners of objects directly. The whole method is end-to-end, does not need any postprocessing steps such as cosine window and bounding box smoothing, thus largely simplifying existing tracking pipelines. The proposed tracker achieves state-of-the-art performance on five challenging short-term and long-term benchmarks, while running at real-time speed, being 6x faster than Siam R-CNN. Code and models are open-sourced at https://github.com/researchmm/Stark.},
 bibtype = {article},
 author = {Yan, Bin and Peng, Houwen and Fu, Jianlong and Wang, Dong and Lu, Huchuan}
}

In this paper, we present a new tracking architecture with an encoder-decoder transformer as the key component. The encoder models the global spatio-temporal feature dependencies between target objects and search regions, while the decoder learns a query embedding to predict the spatial positions of the target objects. Our method casts object tracking as a direct bounding box prediction problem, without using any proposals or predefined anchors. With the encoder-decoder transformer, the prediction of objects just uses a simple fully-convolutional network, which estimates the corners of objects directly. The whole method is end-to-end, does not need any postprocessing steps such as cosine window and bounding box smoothing, thus largely simplifying existing tracking pipelines. The proposed tracker achieves state-of-the-art performance on five challenging short-term and long-term benchmarks, while running at real-time speed, being 6x faster than Siam R-CNN. Code and models are open-sourced at https://github.com/researchmm/Stark.

@article{
 title = {An Empirical Study of Training Self-Supervised Vision Transformers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.02057},
 id = {27a1225c-1187-38aa-832d-48bd5680777b},
 created = {2021-10-14T07:07:44.822Z},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:07:53.205Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b},
 private_publication = {false},
 abstract = {This paper does not describe a novel method. Instead, it studies a straightforward, incremental, yet must-know baseline given the recent progress in computer vision: self-supervised learning for Vision Transformers (ViT). While the training recipes for standard convolutional networks have been highly mature and robust, the recipes for ViT are yet to be built, especially in the self-supervised scenarios where training becomes more challenging. In this work, we go back to basics and investigate the effects of several fundamental components for training self-supervised ViT. We observe that instability is a major issue that degrades accuracy, and it can be hidden by apparently good results. We reveal that these results are indeed partial failure, and they can be improved when training is made more stable. We benchmark ViT results in MoCo v3 and several other self-supervised frameworks, with ablations in various aspects. We discuss the currently positive evidence as well as challenges and open questions. We hope that this work will provide useful data points and experience for future research.},
 bibtype = {article},
 author = {Chen, Xinlei and Xie, Saining and He, Kaiming}
}

This paper does not describe a novel method. Instead, it studies a straightforward, incremental, yet must-know baseline given the recent progress in computer vision: self-supervised learning for Vision Transformers (ViT). While the training recipes for standard convolutional networks have been highly mature and robust, the recipes for ViT are yet to be built, especially in the self-supervised scenarios where training becomes more challenging. In this work, we go back to basics and investigate the effects of several fundamental components for training self-supervised ViT. We observe that instability is a major issue that degrades accuracy, and it can be hidden by apparently good results. We reveal that these results are indeed partial failure, and they can be improved when training is made more stable. We benchmark ViT results in MoCo v3 and several other self-supervised frameworks, with ablations in various aspects. We discuss the currently positive evidence as well as challenges and open questions. We hope that this work will provide useful data points and experience for future research.

@article{
 title = {Understanding Robustness of Transformers for Image Classification},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.14586},
 id = {012e9920-a639-374e-975e-a94c60368931},
 created = {2021-10-14T07:07:44.823Z},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:07:57.970Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b},
 private_publication = {false},
 abstract = {Deep Convolutional Neural Networks (CNNs) have long been the architecture of choice for computer vision tasks. Recently, Transformer-based architectures like Vision Transformer (ViT) have matched or even surpassed ResNets for image classification. However, details of the Transformer architecture -- such as the use of non-overlapping patches -- lead one to wonder whether these networks are as robust. In this paper, we perform an extensive study of a variety of different measures of robustness of ViT models and compare the findings to ResNet baselines. We investigate robustness to input perturbations as well as robustness to model perturbations. We find that when pre-trained with a sufficient amount of data, ViT models are at least as robust as the ResNet counterparts on a broad range of perturbations. We also find that Transformers are robust to the removal of almost any single layer, and that while activations from later layers are highly correlated with each other, they nevertheless play an important role in classification.},
 bibtype = {article},
 author = {Bhojanapalli, Srinadh and Chakrabarti, Ayan and Glasner, Daniel and Li, Daliang and Unterthiner, Thomas and Veit, Andreas}
}

Deep Convolutional Neural Networks (CNNs) have long been the architecture of choice for computer vision tasks. Recently, Transformer-based architectures like Vision Transformer (ViT) have matched or even surpassed ResNets for image classification. However, details of the Transformer architecture -- such as the use of non-overlapping patches -- lead one to wonder whether these networks are as robust. In this paper, we perform an extensive study of a variety of different measures of robustness of ViT models and compare the findings to ResNet baselines. We investigate robustness to input perturbations as well as robustness to model perturbations. We find that when pre-trained with a sufficient amount of data, ViT models are at least as robust as the ResNet counterparts on a broad range of perturbations. We also find that Transformers are robust to the removal of almost any single layer, and that while activations from later layers are highly correlated with each other, they nevertheless play an important role in classification.

@article{
 title = {Vision Transformers for Dense Prediction},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2103.13413},
 id = {d54bb30c-fbc2-371a-9f60-eb9dd23a08e4},
 created = {2021-10-14T07:15:14.737Z},
 accessed = {2021-10-14},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:28:51.438Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b},
 private_publication = {false},
 abstract = {We introduce dense vision transformers, an architecture that leverages vision transformers in place of convolutional networks as a backbone for dense prediction tasks. We assemble tokens from various stages of the vision transformer into image-like representations at various resolutions and progressively combine them into full-resolution predictions using a convolutional decoder. The transformer backbone processes representations at a constant and relatively high resolution and has a global receptive field at every stage. These properties allow the dense vision transformer to provide finer-grained and more globally coherent predictions when compared to fully-convolutional networks. Our experiments show that this architecture yields substantial improvements on dense prediction tasks, especially when a large amount of training data is available. For monocular depth estimation, we observe an improvement of up to 28% in relative performance when compared to a state-of-the-art fully-convolutional network. When applied to semantic segmentation, dense vision transformers set a new state of the art on ADE20K with 49.02% mIoU. We further show that the architecture can be fine-tuned on smaller datasets such as NYUv2, KITTI, and Pascal Context where it also sets the new state of the art. Our models are available at https://github.com/intel-isl/DPT.},
 bibtype = {article},
 author = {Ranftl, René and Bochkovskiy, Alexey and Koltun, Vladlen}
}

We introduce dense vision transformers, an architecture that leverages vision transformers in place of convolutional networks as a backbone for dense prediction tasks. We assemble tokens from various stages of the vision transformer into image-like representations at various resolutions and progressively combine them into full-resolution predictions using a convolutional decoder. The transformer backbone processes representations at a constant and relatively high resolution and has a global receptive field at every stage. These properties allow the dense vision transformer to provide finer-grained and more globally coherent predictions when compared to fully-convolutional networks. Our experiments show that this architecture yields substantial improvements on dense prediction tasks, especially when a large amount of training data is available. For monocular depth estimation, we observe an improvement of up to 28% in relative performance when compared to a state-of-the-art fully-convolutional network. When applied to semantic segmentation, dense vision transformers set a new state of the art on ADE20K with 49.02% mIoU. We further show that the architecture can be fine-tuned on smaller datasets such as NYUv2, KITTI, and Pascal Context where it also sets the new state of the art. Our models are available at https://github.com/intel-isl/DPT.

@article{
 title = {CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification},
 type = {article},
 year = {2021},
 websites = {https://github.com/IBM/CrossViT.},
 id = {bc4f114a-2248-3eea-9831-4d2ae1056813},
 created = {2021-10-14T07:16:56.089Z},
 accessed = {2021-10-14},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-14T07:16:59.053Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {865cda55-bfc4-4a99-88d6-7092e1cbba3b},
 private_publication = {false},
 abstract = {The recently developed vision transformer (ViT) has achieved promising results on image classification compared to convolutional neural networks. Inspired by this, in this paper, we study how to learn multi-scale feature representations in transformer models for image classification. To this end, we propose a dual-branch transformer to combine image patches (i.e., tokens in a transformer) of different sizes to produce stronger image features. Our approach processes small-patch and large-patch tokens with two separate branches of different computational complexity and these tokens are then fused purely by attention multiple times to complement each other. Furthermore, to reduce computation, we develop a simple yet effective token fusion module based on cross attention, which uses a single token for each branch as a query to exchange information with other branches. Our proposed cross-attention only requires linear time for both computational and memory complexity instead of quadratic time otherwise. Extensive experiments demonstrate that our approach performs better than or on par with several concurrent works on vision transformer, in addition to efficient CNN models. For example, on the ImageNet1K dataset, with some architectural changes, our approach outperforms the recent DeiT by a large margin of 2% with a small to moderate increase in FLOPs and model parameters. Our source codes and models are available at https://github.com/IBM/CrossViT.},
 bibtype = {article},
 author = {Chen, Chun-Fu and Fan, Quanfu and Panda, Rameswar}
}

The recently developed vision transformer (ViT) has achieved promising results on image classification compared to convolutional neural networks. Inspired by this, in this paper, we study how to learn multi-scale feature representations in transformer models for image classification. To this end, we propose a dual-branch transformer to combine image patches (i.e., tokens in a transformer) of different sizes to produce stronger image features. Our approach processes small-patch and large-patch tokens with two separate branches of different computational complexity and these tokens are then fused purely by attention multiple times to complement each other. Furthermore, to reduce computation, we develop a simple yet effective token fusion module based on cross attention, which uses a single token for each branch as a query to exchange information with other branches. Our proposed cross-attention only requires linear time for both computational and memory complexity instead of quadratic time otherwise. Extensive experiments demonstrate that our approach performs better than or on par with several concurrent works on vision transformer, in addition to efficient CNN models. For example, on the ImageNet1K dataset, with some architectural changes, our approach outperforms the recent DeiT by a large margin of 2% with a small to moderate increase in FLOPs and model parameters. Our source codes and models are available at https://github.com/IBM/CrossViT.

@article{
 title = {Point cloud classification with deep normalized Reeb graph convolution},
 type = {article},
 year = {2021},
 keywords = {Graph normalization,Point cloud,Reeb graph},
 pages = {104092},
 volume = {106},
 websites = {https://doi.org/10.1016/j.imavis.2020.104092},
 publisher = {Elsevier B.V.},
 id = {167e685d-2a73-3e72-b248-fae628137d49},
 created = {2021-10-22T08:00:15.994Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-11T19:44:51.857Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wang2021},
 private_publication = {false},
 abstract = {Recently, plenty of deep learning methods have been proposed to handle point clouds. Almost all of them input the entire point cloud and ignore the information redundancy lying in point clouds. This paper addresses this problem by extracting the Reeb graph from point clouds, which is a much more informative and compact representation of point clouds, and then filter the graph with deep graph convolution. To be able to classify or segment point clouds well, we propose (1) Graph Normalization to transform various graphs into a canonical graph space; (2) Normalized Similarity Distance to better identify the graph structure;(3) Reeb Graph Guided Node Pooling in order to aggregate high-level features from kNN graphs. Besides, our method naturally fits into the problem of classifying point clouds with unknown orientations. In the results, we show that our method gives a competitive performance to the state-of-the-art methods and outperforms previous methods by a large margin on handling point clouds with unknown orientations.},
 bibtype = {article},
 author = {Wang, Weiming and You, Yang and Liu, Wenhai and Lu, Cewu},
 doi = {10.1016/j.imavis.2020.104092},
 journal = {Image and Vision Computing}
}

Recently, plenty of deep learning methods have been proposed to handle point clouds. Almost all of them input the entire point cloud and ignore the information redundancy lying in point clouds. This paper addresses this problem by extracting the Reeb graph from point clouds, which is a much more informative and compact representation of point clouds, and then filter the graph with deep graph convolution. To be able to classify or segment point clouds well, we propose (1) Graph Normalization to transform various graphs into a canonical graph space; (2) Normalized Similarity Distance to better identify the graph structure;(3) Reeb Graph Guided Node Pooling in order to aggregate high-level features from kNN graphs. Besides, our method naturally fits into the problem of classifying point clouds with unknown orientations. In the results, we show that our method gives a competitive performance to the state-of-the-art methods and outperforms previous methods by a large margin on handling point clouds with unknown orientations.

@article{
 title = {Geometry-Aware Self-Training for Unsupervised Domain Adaptationon Object Point Clouds},
 type = {article},
 year = {2021},
 pages = {6403-6412},
 websites = {http://arxiv.org/abs/2108.09169},
 id = {d66c3327-140e-3367-9a55-9f89d6bd18f2},
 created = {2021-10-30T07:28:03.703Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:18.034Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {The point cloud representation of an object can have a large geometric variation in view of inconsistent data acquisition procedure, which thus leads to domain discrepancy due to diverse and uncontrollable shape representation cross datasets. To improve discrimination on unseen distribution of point-based geometries in a practical and feasible perspective, this paper proposes a new method of geometry-aware self-training (GAST) for unsupervised domain adaptation of object point cloud classification. Specifically, this paper aims to learn a domain-shared representation of semantic categories, via two novel self-supervised geometric learning tasks as feature regularization. On one hand, the representation learning is empowered by a linear mixup of point cloud samples with their self-generated rotation labels, to capture a global topological configuration of local geometries. On the other hand, a diverse point distribution across datasets can be normalized with a novel curvature-aware distortion localization. Experiments on the PointDA-10 dataset show that our GAST method can significantly outperform the state-of-the-art methods.},
 bibtype = {article},
 author = {Zou, Longkun and Tang, Hui and Chen, Ke and Jia, Kui}
}

The point cloud representation of an object can have a large geometric variation in view of inconsistent data acquisition procedure, which thus leads to domain discrepancy due to diverse and uncontrollable shape representation cross datasets. To improve discrimination on unseen distribution of point-based geometries in a practical and feasible perspective, this paper proposes a new method of geometry-aware self-training (GAST) for unsupervised domain adaptation of object point cloud classification. Specifically, this paper aims to learn a domain-shared representation of semantic categories, via two novel self-supervised geometric learning tasks as feature regularization. On one hand, the representation learning is empowered by a linear mixup of point cloud samples with their self-generated rotation labels, to capture a global topological configuration of local geometries. On the other hand, a diverse point distribution across datasets can be normalized with a novel curvature-aware distortion localization. Experiments on the PointDA-10 dataset show that our GAST method can significantly outperform the state-of-the-art methods.

@article{
 title = {Syncretic Modality Collaborative Learning for Visible Infrared Person},
 type = {article},
 year = {2021},
 pages = {225-234},
 id = {b8216c5d-4ea5-349e-9383-beb40dc336c8},
 created = {2021-10-30T07:28:03.751Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:41.394Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 bibtype = {article},
 author = {Iccv, Anonymous and Id, Paper}
}

@article{
 title = {Geometry-based Distance Decomposition for Monocular 3D Object Detection},
 type = {article},
 year = {2021},
 pages = {15172-15181},
 websites = {http://arxiv.org/abs/2104.03775},
 id = {656c9330-b54d-37c8-b6c9-fb3e5bded415},
 created = {2021-10-30T07:28:03.858Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:52.683Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Monocular 3D object detection is of great significance for autonomous driving but remains challenging. The core challenge is to predict the distance of objects in the absence of explicit depth information. Unlike regressing the distance as a single variable in most existing methods, we propose a novel geometry-based distance decomposition to recover the distance by its factors. The decomposition factors the distance of objects into the most representative and stable variables, i.e. the physical height and the projected visual height in the image plane. Moreover, the decomposition maintains the self-consistency between the two heights, leading to robust distance prediction when both predicted heights are inaccurate. The decomposition also enables us to trace the causes of the distance uncertainty for different scenarios. Such decomposition makes the distance prediction interpretable, accurate, and robust. Our method directly predicts 3D bounding boxes from RGB images with a compact architecture, making the training and inference simple and efficient. The experimental results show that our method achieves the state-of-the-art performance on the monocular 3D Object Detection and Birds Eye View tasks of the KITTI dataset, and can generalize to images with different camera intrinsics.},
 bibtype = {article},
 author = {Shi, Xuepeng and Ye, Qi and Chen, Xiaozhi and Chen, Chuangrong and Chen, Zhixiang and Kim, Tae-Kyun}
}

Monocular 3D object detection is of great significance for autonomous driving but remains challenging. The core challenge is to predict the distance of objects in the absence of explicit depth information. Unlike regressing the distance as a single variable in most existing methods, we propose a novel geometry-based distance decomposition to recover the distance by its factors. The decomposition factors the distance of objects into the most representative and stable variables, i.e. the physical height and the projected visual height in the image plane. Moreover, the decomposition maintains the self-consistency between the two heights, leading to robust distance prediction when both predicted heights are inaccurate. The decomposition also enables us to trace the causes of the distance uncertainty for different scenarios. Such decomposition makes the distance prediction interpretable, accurate, and robust. Our method directly predicts 3D bounding boxes from RGB images with a compact architecture, making the training and inference simple and efficient. The experimental results show that our method achieves the state-of-the-art performance on the monocular 3D Object Detection and Birds Eye View tasks of the KITTI dataset, and can generalize to images with different camera intrinsics.

@article{
 title = {Learning with Noisy Labels for Robust Point Cloud Segmentation},
 type = {article},
 year = {2021},
 pages = {6443-6452},
 websites = {http://arxiv.org/abs/2107.14230},
 id = {6358ae46-6d17-39e6-9fe5-3cfcf5d5f5f4},
 created = {2021-10-30T07:28:03.918Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:26.275Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Point cloud segmentation is a fundamental task in 3D. Despite recent progress on point cloud segmentation with the power of deep networks, current deep learning methods based on the clean label assumptions may fail with noisy labels. Yet, object class labels are often mislabeled in real-world point cloud datasets. In this work, we take the lead in solving this issue by proposing a novel Point Noise-Adaptive Learning (PNAL) framework. Compared to existing noise-robust methods on image tasks, our PNAL is noise-rate blind, to cope with the spatially variant noise rate problem specific to point clouds. Specifically, we propose a novel point-wise confidence selection to obtain reliable labels based on the historical predictions of each point. A novel cluster-wise label correction is proposed with a voting strategy to generate the best possible label taking the neighbor point correlations into consideration. We conduct extensive experiments to demonstrate the effectiveness of PNAL on both synthetic and real-world noisy datasets. In particular, even with $60\%$ symmetric noisy labels, our proposed method produces much better results than its baseline counterpart without PNAL and is comparable to the ideal upper bound trained on a completely clean dataset. Moreover, we fully re-labeled the validation set of a popular but noisy real-world scene dataset ScanNetV2 to make it clean, for rigorous experiment and future research. Our code and data are available at \urlhttps://shuquanye.com/PNAL_website/.},
 bibtype = {article},
 author = {Ye, Shuquan and Chen, Dongdong and Han, Songfang and Liao, Jing}
}

Point cloud segmentation is a fundamental task in 3D. Despite recent progress on point cloud segmentation with the power of deep networks, current deep learning methods based on the clean label assumptions may fail with noisy labels. Yet, object class labels are often mislabeled in real-world point cloud datasets. In this work, we take the lead in solving this issue by proposing a novel Point Noise-Adaptive Learning (PNAL) framework. Compared to existing noise-robust methods on image tasks, our PNAL is noise-rate blind, to cope with the spatially variant noise rate problem specific to point clouds. Specifically, we propose a novel point-wise confidence selection to obtain reliable labels based on the historical predictions of each point. A novel cluster-wise label correction is proposed with a voting strategy to generate the best possible label taking the neighbor point correlations into consideration. We conduct extensive experiments to demonstrate the effectiveness of PNAL on both synthetic and real-world noisy datasets. In particular, even with $60\%$ symmetric noisy labels, our proposed method produces much better results than its baseline counterpart without PNAL and is comparable to the ideal upper bound trained on a completely clean dataset. Moreover, we fully re-labeled the validation set of a popular but noisy real-world scene dataset ScanNetV2 to make it clean, for rigorous experiment and future research. Our code and data are available at \urlhttps://shuquanye.com/PNAL_website/.

@article{
 title = {DRINet: A Dual-Representation Iterative Learning Network for Point Cloud Segmentation},
 type = {article},
 year = {2021},
 pages = {7447-7456},
 websites = {http://arxiv.org/abs/2108.04023},
 id = {eca95443-1649-3df7-97ff-17a1b7592623},
 created = {2021-10-30T07:28:03.950Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:28.592Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {We present a novel and flexible architecture for point cloud segmentation with dual-representation iterative learning. In point cloud processing, different representations have their own pros and cons. Thus, finding suitable ways to represent point cloud data structure while keeping its own internal physical property such as permutation and scale-invariant is a fundamental problem. Therefore, we propose our work, DRINet, which serves as the basic network structure for dual-representation learning with great flexibility at feature transferring and less computation cost, especially for large-scale point clouds. DRINet mainly consists of two modules called Sparse Point-Voxel Feature Extraction and Sparse Voxel-Point Feature Extraction. By utilizing these two modules iteratively, features can be propagated between two different representations. We further propose a novel multi-scale pooling layer for pointwise locality learning to improve context information propagation. Our network achieves state-of-the-art results for point cloud classification and segmentation tasks on several datasets while maintaining high runtime efficiency. For large-scale outdoor scenarios, our method outperforms state-of-the-art methods with a real-time inference speed of 62ms per frame.},
 bibtype = {article},
 author = {Ye, Maosheng and Xu, Shuangjie and Cao, Tongyi and Chen, Qifeng}
}

We present a novel and flexible architecture for point cloud segmentation with dual-representation iterative learning. In point cloud processing, different representations have their own pros and cons. Thus, finding suitable ways to represent point cloud data structure while keeping its own internal physical property such as permutation and scale-invariant is a fundamental problem. Therefore, we propose our work, DRINet, which serves as the basic network structure for dual-representation learning with great flexibility at feature transferring and less computation cost, especially for large-scale point clouds. DRINet mainly consists of two modules called Sparse Point-Voxel Feature Extraction and Sparse Voxel-Point Feature Extraction. By utilizing these two modules iteratively, features can be propagated between two different representations. We further propose a novel multi-scale pooling layer for pointwise locality learning to improve context information propagation. Our network achieves state-of-the-art results for point cloud classification and segmentation tasks on several datasets while maintaining high runtime efficiency. For large-scale outdoor scenarios, our method outperforms state-of-the-art methods with a real-time inference speed of 62ms per frame.

@article{
 title = {Learning Meta-class Memory for Few-Shot Semantic Segmentation},
 type = {article},
 year = {2021},
 pages = {517-526},
 websites = {http://arxiv.org/abs/2108.02958},
 id = {d5e3e8d3-4676-3568-8646-52a9df7fa357},
 created = {2021-10-30T07:28:03.974Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:39.005Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Currently, the state-of-the-art methods treat few-shot semantic segmentation task as a conditional foreground-background segmentation problem, assuming each class is independent. In this paper, we introduce the concept of meta-class, which is the meta information (e.g. certain middle-level features) shareable among all classes. To explicitly learn meta-class representations in few-shot segmentation task, we propose a novel Meta-class Memory based few-shot segmentation method (MM-Net), where we introduce a set of learnable memory embeddings to memorize the meta-class information during the base class training and transfer to novel classes during the inference stage. Moreover, for the $k$-shot scenario, we propose a novel image quality measurement module to select images from the set of support images. A high-quality class prototype could be obtained with the weighted sum of support image features based on the quality measure. Experiments on both PASCAL-$5^i$ and COCO dataset shows that our proposed method is able to achieve state-of-the-art results in both 1-shot and 5-shot settings. Particularly, our proposed MM-Net achieves 37.5\% mIoU on the COCO dataset in 1-shot setting, which is 5.1\% higher than the previous state-of-the-art.},
 bibtype = {article},
 author = {Wu, Zhonghua and Shi, Xiangxi and lin, Guosheng and Cai, Jianfei}
}

Currently, the state-of-the-art methods treat few-shot semantic segmentation task as a conditional foreground-background segmentation problem, assuming each class is independent. In this paper, we introduce the concept of meta-class, which is the meta information (e.g. certain middle-level features) shareable among all classes. To explicitly learn meta-class representations in few-shot segmentation task, we propose a novel Meta-class Memory based few-shot segmentation method (MM-Net), where we introduce a set of learnable memory embeddings to memorize the meta-class information during the base class training and transfer to novel classes during the inference stage. Moreover, for the $k$-shot scenario, we propose a novel image quality measurement module to select images from the set of support images. A high-quality class prototype could be obtained with the weighted sum of support image features based on the quality measure. Experiments on both PASCAL-$5^i$ and COCO dataset shows that our proposed method is able to achieve state-of-the-art results in both 1-shot and 5-shot settings. Particularly, our proposed MM-Net achieves 37.5\% mIoU on the COCO dataset in 1-shot setting, which is 5.1\% higher than the previous state-of-the-art.

@article{
 title = {Towers of Babel: Combining Images, Language, and 3D Geometry for Learning Multimodal Vision},
 type = {article},
 year = {2021},
 pages = {428-437},
 websites = {http://arxiv.org/abs/2108.05863},
 id = {80757982-55b9-3e58-bc4a-0b87850dec62},
 created = {2021-10-30T07:28:03.974Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:36.896Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {The abundance and richness of Internet photos of landmarks and cities has led to significant progress in 3D vision over the past two decades, including automated 3D reconstructions of the world's landmarks from tourist photos. However, a major source of information available for these 3D-augmented collections---namely language, e.g., from image captions---has been virtually untapped. In this work, we present WikiScenes, a new, large-scale dataset of landmark photo collections that contains descriptive text in the form of captions and hierarchical category names. WikiScenes forms a new testbed for multimodal reasoning involving images, text, and 3D geometry. We demonstrate the utility of WikiScenes for learning semantic concepts over images and 3D models. Our weakly-supervised framework connects images, 3D structure, and semantics---utilizing the strong constraints provided by 3D geometry---to associate semantic concepts to image pixels and 3D points.},
 bibtype = {article},
 author = {Wu, Xiaoshi and Averbuch-Elor, Hadar and Sun, Jin and Snavely, Noah}
}

The abundance and richness of Internet photos of landmarks and cities has led to significant progress in 3D vision over the past two decades, including automated 3D reconstructions of the world's landmarks from tourist photos. However, a major source of information available for these 3D-augmented collections---namely language, e.g., from image captions---has been virtually untapped. In this work, we present WikiScenes, a new, large-scale dataset of landmark photo collections that contains descriptive text in the form of captions and hierarchical category names. WikiScenes forms a new testbed for multimodal reasoning involving images, text, and 3D geometry. We demonstrate the utility of WikiScenes for learning semantic concepts over images and 3D models. Our weakly-supervised framework connects images, 3D structure, and semantics---utilizing the strong constraints provided by 3D geometry---to associate semantic concepts to image pixels and 3D points.

@article{
 title = {CPFN: Cascaded Primitive Fitting Networks for High-Resolution Point Clouds},
 type = {article},
 year = {2021},
 pages = {7457-7466},
 websites = {http://arxiv.org/abs/2109.00113},
 id = {28fc7436-cc11-3c00-92ed-fc7f9e54f91c},
 created = {2021-10-30T07:28:04.150Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:29:24.954Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Representing human-made objects as a collection of base primitives has a long history in computer vision and reverse engineering. In the case of high-resolution point cloud scans, the challenge is to be able to detect both large primitives as well as those explaining the detailed parts. While the classical RANSAC approach requires case-specific parameter tuning, state-of-the-art networks are limited by memory consumption of their backbone modules such as PointNet++, and hence fail to detect the fine-scale primitives. We present Cascaded Primitive Fitting Networks (CPFN) that relies on an adaptive patch sampling network to assemble detection results of global and local primitive detection networks. As a key enabler, we present a merging formulation that dynamically aggregates the primitives across global and local scales. Our evaluation demonstrates that CPFN improves the state-of-the-art SPFN performance by 13-14% on high-resolution point cloud datasets and specifically improves the detection of fine-scale primitives by 20-22%.},
 bibtype = {article},
 author = {Lê, Eric-Tuan and Sung, Minhyuk and Ceylan, Duygu and Mech, Radomir and Boubekeur, Tamy and Mitra, Niloy J.}
}

Representing human-made objects as a collection of base primitives has a long history in computer vision and reverse engineering. In the case of high-resolution point cloud scans, the challenge is to be able to detect both large primitives as well as those explaining the detailed parts. While the classical RANSAC approach requires case-specific parameter tuning, state-of-the-art networks are limited by memory consumption of their backbone modules such as PointNet++, and hence fail to detect the fine-scale primitives. We present Cascaded Primitive Fitting Networks (CPFN) that relies on an adaptive patch sampling network to assemble detection results of global and local primitive detection networks. As a key enabler, we present a merging formulation that dynamically aggregates the primitives across global and local scales. Our evaluation demonstrates that CPFN improves the state-of-the-art SPFN performance by 13-14% on high-resolution point cloud datasets and specifically improves the detection of fine-scale primitives by 20-22%.

@article{
 title = {Joint Representation Learning and Novel Category Discovery on Single- and Multi-modal Data},
 type = {article},
 year = {2021},
 pages = {610-619},
 websites = {http://arxiv.org/abs/2104.12673},
 id = {d157b2c8-a64c-371b-83b6-e430d7b713db},
 created = {2021-10-30T07:28:04.181Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:32:04.265Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {This paper studies the problem of novel category discovery on single- and multi-modal data with labels from different but relevant categories. We present a generic, end-to-end framework to jointly learn a reliable representation and assign clusters to unlabelled data. To avoid over-fitting the learnt embedding to labelled data, we take inspiration from self-supervised representation learning by noise-contrastive estimation and extend it to jointly handle labelled and unlabelled data. In particular, we propose using category discrimination on labelled data and cross-modal discrimination on multi-modal data to augment instance discrimination used in conventional contrastive learning approaches. We further employ Winner-Take-All (WTA) hashing algorithm on the shared representation space to generate pairwise pseudo labels for unlabelled data to better predict cluster assignments. We thoroughly evaluate our framework on large-scale multi-modal video benchmarks Kinetics-400 and VGG-Sound, and image benchmarks CIFAR10, CIFAR100 and ImageNet, obtaining state-of-the-art results.},
 bibtype = {article},
 author = {Jia, Xuhui and Han, Kai and Zhu, Yukun and Green, Bradley}
}

This paper studies the problem of novel category discovery on single- and multi-modal data with labels from different but relevant categories. We present a generic, end-to-end framework to jointly learn a reliable representation and assign clusters to unlabelled data. To avoid over-fitting the learnt embedding to labelled data, we take inspiration from self-supervised representation learning by noise-contrastive estimation and extend it to jointly handle labelled and unlabelled data. In particular, we propose using category discrimination on labelled data and cross-modal discrimination on multi-modal data to augment instance discrimination used in conventional contrastive learning approaches. We further employ Winner-Take-All (WTA) hashing algorithm on the shared representation space to generate pairwise pseudo labels for unlabelled data to better predict cluster assignments. We thoroughly evaluate our framework on large-scale multi-modal video benchmarks Kinetics-400 and VGG-Sound, and image benchmarks CIFAR10, CIFAR100 and ImageNet, obtaining state-of-the-art results.

@article{
 title = {Learning Inner-Group Relations on Point Clouds},
 type = {article},
 year = {2021},
 pages = {15477-15487},
 websites = {http://arxiv.org/abs/2108.12468},
 id = {df10465c-85a6-3ecf-a4b3-e576fe3c3b88},
 created = {2021-10-30T07:28:04.287Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:28:54.492Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {The prevalence of relation networks in computer vision is in stark contrast to underexplored point-based methods. In this paper, we explore the possibilities of local relation operators and survey their feasibility. We propose a scalable and efficient module, called group relation aggregator. The module computes a feature of a group based on the aggregation of the features of the inner-group points weighted by geometric relations and semantic relations. We adopt this module to design our RPNet. We further verify the expandability of RPNet, in terms of both depth and width, on the tasks of classification and segmentation. Surprisingly, empirical results show that wider RPNet fits for classification, while deeper RPNet works better on segmentation. RPNet achieves state-of-the-art for classification and segmentation on challenging benchmarks. We also compare our local aggregator with PointNet++, with around 30% parameters and 50% computation saving. Finally, we conduct experiments to reveal the robustness of RPNet with regard to rigid transformation and noises.},
 bibtype = {article},
 author = {Ran, Haoxi and Zhuo, Wei and Liu, Jun and Lu, Li}
}

The prevalence of relation networks in computer vision is in stark contrast to underexplored point-based methods. In this paper, we explore the possibilities of local relation operators and survey their feasibility. We propose a scalable and efficient module, called group relation aggregator. The module computes a feature of a group based on the aggregation of the features of the inner-group points weighted by geometric relations and semantic relations. We adopt this module to design our RPNet. We further verify the expandability of RPNet, in terms of both depth and width, on the tasks of classification and segmentation. Surprisingly, empirical results show that wider RPNet fits for classification, while deeper RPNet works better on segmentation. RPNet achieves state-of-the-art for classification and segmentation on challenging benchmarks. We also compare our local aggregator with PointNet++, with around 30% parameters and 50% computation saving. Finally, we conduct experiments to reveal the robustness of RPNet with regard to rigid transformation and noises.

@article{
 title = {LSG-CPD: Coherent Point Drift with Local Surface Geometry for Point Cloud Registration},
 type = {article},
 year = {2021},
 pages = {15293-15302},
 websites = {http://arxiv.org/abs/2103.15039},
 id = {e4597dcb-2bc2-3484-86ad-835ee28a4537},
 created = {2021-10-30T07:28:04.507Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:29:01.268Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Probabilistic point cloud registration methods are becoming more popular because of their robustness. However, unlike point-to-plane variants of iterative closest point (ICP) which incorporate local surface geometric information such as surface normals, most probabilistic methods (e.g., coherent point drift (CPD)) ignore such information and build Gaussian mixture models (GMMs) with isotropic Gaussian covariances. This results in sphere-like GMM components which only penalize the point-to-point distance between the two point clouds. In this paper, we propose a novel method called CPD with Local Surface Geometry (LSG-CPD) for rigid point cloud registration. Our method adaptively adds different levels of point-to-plane penalization on top of the point-to-point penalization based on the flatness of the local surface. This results in GMM components with anisotropic covariances. We formulate point cloud registration as a maximum likelihood estimation (MLE) problem and solve it with the Expectation-Maximization (EM) algorithm. In the E step, we demonstrate that the computation can be recast into simple matrix manipulations and efficiently computed on a GPU. In the M step, we perform an unconstrained optimization on a matrix Lie group to efficiently update the rigid transformation of the registration. The proposed method outperforms state-of-the-art algorithms in terms of accuracy and robustness on various datasets captured with range scanners, RGBD cameras, and LiDARs. Also, it is significantly faster than modern implementations of CPD. The code will be released.},
 bibtype = {article},
 author = {Liu, Weixiao and Wu, Hongtao and Chirikjian, Gregory}
}

Probabilistic point cloud registration methods are becoming more popular because of their robustness. However, unlike point-to-plane variants of iterative closest point (ICP) which incorporate local surface geometric information such as surface normals, most probabilistic methods (e.g., coherent point drift (CPD)) ignore such information and build Gaussian mixture models (GMMs) with isotropic Gaussian covariances. This results in sphere-like GMM components which only penalize the point-to-point distance between the two point clouds. In this paper, we propose a novel method called CPD with Local Surface Geometry (LSG-CPD) for rigid point cloud registration. Our method adaptively adds different levels of point-to-plane penalization on top of the point-to-point penalization based on the flatness of the local surface. This results in GMM components with anisotropic covariances. We formulate point cloud registration as a maximum likelihood estimation (MLE) problem and solve it with the Expectation-Maximization (EM) algorithm. In the E step, we demonstrate that the computation can be recast into simple matrix manipulations and efficiently computed on a GPU. In the M step, we perform an unconstrained optimization on a matrix Lie group to efficiently update the rigid transformation of the registration. The proposed method outperforms state-of-the-art algorithms in terms of accuracy and robustness on various datasets captured with range scanners, RGBD cameras, and LiDARs. Also, it is significantly faster than modern implementations of CPD. The code will be released.

@article{
 title = {Towards Efficient Graph Convolutional Networks for Point Cloud Handling},
 type = {article},
 year = {2021},
 pages = {3752-3762},
 websites = {http://arxiv.org/abs/2104.05706},
 id = {7af17445-bb34-368b-98fe-6ec107796dc0},
 created = {2021-10-30T07:28:04.637Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:29:09.097Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {In this paper, we aim at improving the computational efficiency of graph convolutional networks (GCNs) for learning on point clouds. The basic graph convolution that is typically composed of a $K$-nearest neighbor (KNN) search and a multilayer perceptron (MLP) is examined. By mathematically analyzing the operations there, two findings to improve the efficiency of GCNs are obtained. (1) The local geometric structure information of 3D representations propagates smoothly across the GCN that relies on KNN search to gather neighborhood features. This motivates the simplification of multiple KNN searches in GCNs. (2) Shuffling the order of graph feature gathering and an MLP leads to equivalent or similar composite operations. Based on those findings, we optimize the computational procedure in GCNs. A series of experiments show that the optimized networks have reduced computational complexity, decreased memory consumption, and accelerated inference speed while maintaining comparable accuracy for learning on point clouds. Code will be available at \urlhttps://github.com/ofsoundof/EfficientGCN.git.},
 bibtype = {article},
 author = {Li, Yawei and Chen, He and Cui, Zhaopeng and Timofte, Radu and Pollefeys, Marc and Chirikjian, Gregory and Van Gool, Luc}
}

In this paper, we aim at improving the computational efficiency of graph convolutional networks (GCNs) for learning on point clouds. The basic graph convolution that is typically composed of a $K$-nearest neighbor (KNN) search and a multilayer perceptron (MLP) is examined. By mathematically analyzing the operations there, two findings to improve the efficiency of GCNs are obtained. (1) The local geometric structure information of 3D representations propagates smoothly across the GCN that relies on KNN search to gather neighborhood features. This motivates the simplification of multiple KNN searches in GCNs. (2) Shuffling the order of graph feature gathering and an MLP leads to equivalent or similar composite operations. Based on those findings, we optimize the computational procedure in GCNs. A series of experiments show that the optimized networks have reduced computational complexity, decreased memory consumption, and accelerated inference speed while maintaining comparable accuracy for learning on point clouds. Code will be available at \urlhttps://github.com/ofsoundof/EfficientGCN.git.

@article{
 title = {LIGA-Stereo: Learning LiDAR Geometry Aware Representations for Stereo-based 3D Detector},
 type = {article},
 year = {2021},
 pages = {3153-3163},
 websites = {http://arxiv.org/abs/2108.08258},
 id = {9a3897ac-b5e1-39f0-9117-207b0421e502},
 created = {2021-10-30T07:28:04.772Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-10-30T07:32:13.653Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {48c2017d-ceec-4fcb-b966-b19e6f311352},
 private_publication = {false},
 abstract = {Stereo-based 3D detection aims at detecting 3D object bounding boxes from stereo images using intermediate depth maps or implicit 3D geometry representations, which provides a low-cost solution for 3D perception. However, its performance is still inferior compared with LiDAR-based detection algorithms. To detect and localize accurate 3D bounding boxes, LiDAR-based models can encode accurate object boundaries and surface normal directions from LiDAR point clouds. However, the detection results of stereo-based detectors are easily affected by the erroneous depth features due to the limitation of stereo matching. To solve the problem, we propose LIGA-Stereo (LiDAR Geometry Aware Stereo Detector) to learn stereo-based 3D detectors under the guidance of high-level geometry-aware representations of LiDAR-based detection models. In addition, we found existing voxel-based stereo detectors failed to learn semantic features effectively from indirect 3D supervisions. We attach an auxiliary 2D detection head to provide direct 2D semantic supervisions. Experiment results show that the above two strategies improved the geometric and semantic representation capabilities. Compared with the state-of-the-art stereo detector, our method has improved the 3D detection performance of cars, pedestrians, cyclists by 10.44%, 5.69%, 5.97% mAP respectively on the official KITTI benchmark. The gap between stereo-based and LiDAR-based 3D detectors is further narrowed.},
 bibtype = {article},
 author = {Guo, Xiaoyang and Shi, Shaoshuai and Wang, Xiaogang and Li, Hongsheng}
}

Stereo-based 3D detection aims at detecting 3D object bounding boxes from stereo images using intermediate depth maps or implicit 3D geometry representations, which provides a low-cost solution for 3D perception. However, its performance is still inferior compared with LiDAR-based detection algorithms. To detect and localize accurate 3D bounding boxes, LiDAR-based models can encode accurate object boundaries and surface normal directions from LiDAR point clouds. However, the detection results of stereo-based detectors are easily affected by the erroneous depth features due to the limitation of stereo matching. To solve the problem, we propose LIGA-Stereo (LiDAR Geometry Aware Stereo Detector) to learn stereo-based 3D detectors under the guidance of high-level geometry-aware representations of LiDAR-based detection models. In addition, we found existing voxel-based stereo detectors failed to learn semantic features effectively from indirect 3D supervisions. We attach an auxiliary 2D detection head to provide direct 2D semantic supervisions. Experiment results show that the above two strategies improved the geometric and semantic representation capabilities. Compared with the state-of-the-art stereo detector, our method has improved the 3D detection performance of cars, pedestrians, cyclists by 10.44%, 5.69%, 5.97% mAP respectively on the official KITTI benchmark. The gap between stereo-based and LiDAR-based 3D detectors is further narrowed.

@article{
 title = {Two Heads are Better than One : Geometric-Latent Attention for Point Cloud Segmentation},
 type = {article},
 year = {2021},
 pages = {1-14},
 id = {49ac1c29-d2f6-3023-9647-3accaa66637b},
 created = {2021-11-23T08:03:05.325Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:03:20.837Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {We present an innovative two-headed attention layer that combines geometric and latent features to segment a 3D scene into semantically meaningful subsets. Each head combines local and global information, using either the geometric or latent features, of a neighborhood of points and uses this information to learn better local relationships. This Geometric-Latent attention layer (Ge-Latto) is combined with a sub-sampling strategy to capture global features. Our method is invariant to permutation thanks to the use of shared-MLP layers, and it can also be used with point clouds with varying densities because the local attention layer does not depend on the neighbor order. Our proposal is simple yet robust, which allows it to achieve competitive results in the ShapeNetPart and ModelNet40 datasets, and the state-of-the-art when segmenting the complex dataset S3DIS, with 69.2% IoU on Area 5, and 89.7% overall accuracy using K-fold cross-validation on the 6 areas.},
 bibtype = {article},
 author = {Attention, Geometric-latent}
}

We present an innovative two-headed attention layer that combines geometric and latent features to segment a 3D scene into semantically meaningful subsets. Each head combines local and global information, using either the geometric or latent features, of a neighborhood of points and uses this information to learn better local relationships. This Geometric-Latent attention layer (Ge-Latto) is combined with a sub-sampling strategy to capture global features. Our method is invariant to permutation thanks to the use of shared-MLP layers, and it can also be used with point clouds with varying densities because the local attention layer does not depend on the neighbor order. Our proposal is simple yet robust, which allows it to achieve competitive results in the ShapeNetPart and ModelNet40 datasets, and the state-of-the-art when segmenting the complex dataset S3DIS, with 69.2% IoU on Area 5, and 89.7% overall accuracy using K-fold cross-validation on the 6 areas.

@article{
 title = {Self-Supervised Point Cloud Completion via Inpainting},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.10701},
 id = {ae68f287-15f6-3644-844a-bd31b2e0a17b},
 created = {2021-11-23T08:03:05.326Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:03:35.248Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {When navigating in urban environments, many of the objects that need to be tracked and avoided are heavily occluded. Planning and tracking using these partial scans can be challenging. The aim of this work is to learn to complete these partial point clouds, giving us a full understanding of the object's geometry using only partial observations. Previous methods achieve this with the help of complete, ground-truth annotations of the target objects, which are available only for simulated datasets. However, such ground truth is unavailable for real-world LiDAR data. In this work, we present a self-supervised point cloud completion algorithm, PointPnCNet, which is trained only on partial scans without assuming access to complete, ground-truth annotations. Our method achieves this via inpainting. We remove a portion of the input data and train the network to complete the missing region. As it is difficult to determine which regions were occluded in the initial cloud and which were synthetically removed, our network learns to complete the full cloud, including the missing regions in the initial partial cloud. We show that our method outperforms previous unsupervised and weakly-supervised methods on both the synthetic dataset, ShapeNet, and real-world LiDAR dataset, Semantic KITTI.},
 bibtype = {article},
 author = {Mittal, Himangi and Okorn, Brian and Jangid, Arpit and Held, David}
}

When navigating in urban environments, many of the objects that need to be tracked and avoided are heavily occluded. Planning and tracking using these partial scans can be challenging. The aim of this work is to learn to complete these partial point clouds, giving us a full understanding of the object's geometry using only partial observations. Previous methods achieve this with the help of complete, ground-truth annotations of the target objects, which are available only for simulated datasets. However, such ground truth is unavailable for real-world LiDAR data. In this work, we present a self-supervised point cloud completion algorithm, PointPnCNet, which is trained only on partial scans without assuming access to complete, ground-truth annotations. Our method achieves this via inpainting. We remove a portion of the input data and train the network to complete the missing region. As it is difficult to determine which regions were occluded in the initial cloud and which were synthetically removed, our network learns to complete the full cloud, including the missing regions in the initial partial cloud. We show that our method outperforms previous unsupervised and weakly-supervised methods on both the synthetic dataset, ShapeNet, and real-world LiDAR dataset, Semantic KITTI.

@article{
 title = {Adversarial Robustness Comparison of Vision Transformer and MLP-Mixer to CNNs},
 type = {article},
 year = {2021},
 pages = {1-16},
 websites = {http://arxiv.org/abs/2110.02797},
 id = {629ad79f-a3c3-32e0-8c2e-0b0cb94c0c05},
 created = {2021-11-23T08:03:05.444Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:03:31.380Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model architectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adversarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations.},
 bibtype = {article},
 author = {Benz, Philipp and Ham, Soomin and Zhang, Chaoning and Karjauv, Adil and Kweon, In So}
}

Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model architectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adversarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations.

@article{
 title = {Adversarial Graph Convolutional Network for 3D Point Cloud Segmentation},
 type = {article},
 year = {2021},
 id = {710f421d-bf1b-3d72-ac0e-62ac658b037f},
 created = {2021-11-23T08:03:05.458Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:50.106Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8315fdc0-e3a9-47f0-9186-21b3433d86d2,d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 bibtype = {article},
 author = {Guidelines, Bmvc Author}
}

@article{
 title = {Multi-Modality Task Cascade for 3D Object Detection},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.04013},
 id = {00330d4d-561e-3ac1-863c-19b2e15e4039},
 created = {2021-11-23T08:03:05.469Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:11:31.086Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {Point clouds and RGB images are naturally complementary modalities for 3D visual understanding - the former provides sparse but accurate locations of points on objects, while the latter contains dense color and texture information. Despite this potential for close sensor fusion, many methods train two models in isolation and use simple feature concatenation to represent 3D sensor data. This separated training scheme results in potentially sub-optimal performance and prevents 3D tasks from being used to benefit 2D tasks that are often useful on their own. To provide a more integrated approach, we propose a novel Multi-Modality Task Cascade network (MTC-RCNN) that leverages 3D box proposals to improve 2D segmentation predictions, which are then used to further refine the 3D boxes. We show that including a 2D network between two stages of 3D modules significantly improves both 2D and 3D task performance. Moreover, to prevent the 3D module from over-relying on the overfitted 2D predictions, we propose a dual-head 2D segmentation training and inference scheme, allowing the 2nd 3D module to learn to interpret imperfect 2D segmentation predictions. Evaluating our model on the challenging SUN RGB-D dataset, we improve upon state-of-the-art results of both single modality and fusion networks by a large margin ($\textbf+3.8$ mAP@0.5). Code will be released $\hrefhttps://github.com/Divadi/MTC_RCNN\texthere.$},
 bibtype = {article},
 author = {Park, Jinhyung and Weng, Xinshuo and Man, Yunze and Kitani, Kris}
}

Point clouds and RGB images are naturally complementary modalities for 3D visual understanding - the former provides sparse but accurate locations of points on objects, while the latter contains dense color and texture information. Despite this potential for close sensor fusion, many methods train two models in isolation and use simple feature concatenation to represent 3D sensor data. This separated training scheme results in potentially sub-optimal performance and prevents 3D tasks from being used to benefit 2D tasks that are often useful on their own. To provide a more integrated approach, we propose a novel Multi-Modality Task Cascade network (MTC-RCNN) that leverages 3D box proposals to improve 2D segmentation predictions, which are then used to further refine the 3D boxes. We show that including a 2D network between two stages of 3D modules significantly improves both 2D and 3D task performance. Moreover, to prevent the 3D module from over-relying on the overfitted 2D predictions, we propose a dual-head 2D segmentation training and inference scheme, allowing the 2nd 3D module to learn to interpret imperfect 2D segmentation predictions. Evaluating our model on the challenging SUN RGB-D dataset, we improve upon state-of-the-art results of both single modality and fusion networks by a large margin ($\textbf+3.8$ mAP@0.5). Code will be released $\hrefhttps://github.com/Divadi/MTC_RCNN\texthere.$

@article{
 title = {Planar Shape Based Registration for Multi-modal Geometry},
 type = {article},
 year = {2021},
 pages = {1-15},
 id = {fe87832f-0b45-33d6-a9b9-4348e1f0ce6f},
 created = {2021-11-23T08:03:05.475Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:49.910Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 bibtype = {article},
 author = {Li, Muxingzi and Antipolis, Sophia}
}

@article{
 title = {Enhancing Local Feature Learning for 3D Point Cloud Processing using Unary-Pairwise Attention},
 type = {article},
 year = {2021},
 pages = {1-14},
 id = {c309ed76-8c05-3121-a69b-8741bfce1707},
 created = {2021-11-23T08:03:05.491Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:06:36.611Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 bibtype = {article},
 author = {Xiu, Haoyi}
}

@article{
 title = {VIN: Voxel-based Implicit Network for Joint 3D Object Detection and Segmentation for Lidars},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.02980},
 id = {fead10e1-ad6e-3512-a1fa-649a1aef2866},
 created = {2021-11-23T08:03:05.599Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:03:13.376Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {A unified neural network structure is presented for joint 3D object detection and point cloud segmentation in this paper. We leverage rich supervision from both detection and segmentation labels rather than using just one of them. In addition, an extension based on single-stage object detectors is proposed based on the implicit function widely used in 3D scene and object understanding. The extension branch takes the final feature map from the object detection module as input, and produces an implicit function that generates semantic distribution for each point for its corresponding voxel center. We demonstrated the performance of our structure on nuScenes-lidarseg, a large-scale outdoor dataset. Our solution achieves competitive results against state-of-the-art methods in both 3D object detection and point cloud segmentation with little additional computation load compared with object detection solutions. The capability of efficient weakly supervision semantic segmentation of the proposed method is also validated by experiments.},
 bibtype = {article},
 author = {Zhong, Yuanxin and Zhu, Minghan and Peng, Huei}
}

A unified neural network structure is presented for joint 3D object detection and point cloud segmentation in this paper. We leverage rich supervision from both detection and segmentation labels rather than using just one of them. In addition, an extension based on single-stage object detectors is proposed based on the implicit function widely used in 3D scene and object understanding. The extension branch takes the final feature map from the object detection module as input, and produces an implicit function that generates semantic distribution for each point for its corresponding voxel center. We demonstrated the performance of our structure on nuScenes-lidarseg, a large-scale outdoor dataset. Our solution achieves competitive results against state-of-the-art methods in both 3D object detection and point cloud segmentation with little additional computation load compared with object detection solutions. The capability of efficient weakly supervision semantic segmentation of the proposed method is also validated by experiments.

@article{
 title = {2 . 5D-VoteNet : Depth Map based 3D Object Detection for Real-Time Applications},
 type = {article},
 year = {2021},
 id = {8a365847-9980-38c7-81e5-9f0686137525},
 created = {2021-11-23T08:03:05.603Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:51:28.691Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {bc1835e2-32e3-4f2a-b03c-9540bbbd02e0,d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 bibtype = {article},
 author = {Li, Lanxiao and Heizmann, Michael}
}

@article{
 title = {Cascading Feature Extraction for Fast Point Cloud Registration},
 type = {article},
 year = {2021},
 pages = {1-12},
 websites = {http://arxiv.org/abs/2110.12204},
 id = {62be2787-e332-3ea2-9d79-30e03bbaae22},
 created = {2021-11-23T08:03:05.626Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:06:24.136Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 abstract = {We propose a method for speeding up a 3D point cloud registration through a cascading feature extraction. The current approach with the highest accuracy is realized by iteratively executing feature extraction and registration using deep features. However, iterative feature extraction takes time. Our proposed method significantly reduces the computational cost using cascading shallow layers. Our idea is to omit redundant computations that do not always contribute to the final accuracy. The proposed approach is approximately three times faster than the existing methods without a loss of accuracy.},
 bibtype = {article},
 author = {Hisadome, Yoichiro and Matsui, Yusuke}
}

We propose a method for speeding up a 3D point cloud registration through a cascading feature extraction. The current approach with the highest accuracy is realized by iteratively executing feature extraction and registration using deep features. However, iterative feature extraction takes time. Our proposed method significantly reduces the computational cost using cascading shallow layers. Our idea is to omit redundant computations that do not always contribute to the final accuracy. The proposed approach is approximately three times faster than the existing methods without a loss of accuracy.

@article{
 title = {Adaptive GMM Convolution for Point Cloud Learning},
 type = {article},
 year = {2021},
 pages = {1-13},
 id = {d3276b45-4ff9-346a-8a83-366c04d60863},
 created = {2021-11-23T08:03:05.696Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:49.703Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {d0ca493b-f12a-45f6-9df1-0ad1fe3a78ff},
 private_publication = {false},
 bibtype = {article},
 author = {Wang, Huan}
}

@article{
 title = {SwinFGHash: Fine-grained Image Retrieval via Transformer-based Hashing Network},
 type = {article},
 year = {2021},
 id = {c0e0c8d1-ce73-377c-b170-0f8dd57f1241},
 created = {2021-11-23T08:14:06.940Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-25T08:42:20.916Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Fine-grained image retrieval is a fundamental and challenging problem in computer vision due to the intra-class diversities and inter-class confusions. Existing hashing-based approaches employed convolutional neural networks (CNNs) to learn hash codes for fast fine-grained image retrieval, which are limited by the inherent locality constrain of the convolution operations and yield sub-optimal performance. Recently, transformers have shown colossal potential on vision tasks for their excellent capacity to capture long-range visual dependencies. Therefore, in this paper, we take the first step to exploit the vision transformer-based hashing network for fine-grained image retrieval. We propose the SwinFGHash, which takes advantage of transformer-based architecture to model the feature interactions among the spatially distant areas, e.g., the head and the tail of a bird on an image, thus improving the fine-grained discrimination of the generated hash codes. Besides, we enhance the critical region localization ability of SwinFGHash by designing a Global with Local (GwL) feature learning module, which preserves subtle yet discriminative features for fine-grained retrieval. Extensive experiments on benchmark datasets show that our SwinFGHash significantly outperforms existing state-of-the-art baselines in fine-grained image retrieval.},
 bibtype = {article},
 author = {Lu, Di and Wang, Jinpeng and Zeng, Ziyun and Chen, Bin and Wu, Shudeng and Xia, Shu-Tao}
}

Fine-grained image retrieval is a fundamental and challenging problem in computer vision due to the intra-class diversities and inter-class confusions. Existing hashing-based approaches employed convolutional neural networks (CNNs) to learn hash codes for fast fine-grained image retrieval, which are limited by the inherent locality constrain of the convolution operations and yield sub-optimal performance. Recently, transformers have shown colossal potential on vision tasks for their excellent capacity to capture long-range visual dependencies. Therefore, in this paper, we take the first step to exploit the vision transformer-based hashing network for fine-grained image retrieval. We propose the SwinFGHash, which takes advantage of transformer-based architecture to model the feature interactions among the spatially distant areas, e.g., the head and the tail of a bird on an image, thus improving the fine-grained discrimination of the generated hash codes. Besides, we enhance the critical region localization ability of SwinFGHash by designing a Global with Local (GwL) feature learning module, which preserves subtle yet discriminative features for fine-grained retrieval. Extensive experiments on benchmark datasets show that our SwinFGHash significantly outperforms existing state-of-the-art baselines in fine-grained image retrieval.

@article{
 title = {MVT: Multi-view Vision Transformer for 3D Object Recognition},
 type = {article},
 year = {2021},
 id = {f967c32a-c991-3a4f-b402-0e5f6fd02620},
 created = {2021-11-23T08:15:46.220Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:15:48.667Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Inspired by the great success achieved by CNN in image recognition, view-based methods applied CNNs to model the projected views for 3D object understanding and achieved excellent performance. Nevertheless, multi-view CNN models cannot model the communications between patches from different views, limiting its effectiveness in 3D object recognition. Inspired by the recent success gained by vision Transformer in image recognition, we propose a Multi-view Vision Transformer (MVT) for 3D object recognition. Since each patch feature in a Transformer block has a global reception field, it naturally achieves communications between patches from different views. Meanwhile, it takes much less inductive bias compared with its CNN counterparts. Considering both effectiveness and efficiency, we develop a global-local structure for our MVT. Our experiments on two public benchmarks, ModelNet40 and ModelNet10, demonstrate the competitive performance of our MVT.},
 bibtype = {article},
 author = {Chen, Shuo and Yu, Tan and Li, Ping}
}

Inspired by the great success achieved by CNN in image recognition, view-based methods applied CNNs to model the projected views for 3D object understanding and achieved excellent performance. Nevertheless, multi-view CNN models cannot model the communications between patches from different views, limiting its effectiveness in 3D object recognition. Inspired by the recent success gained by vision Transformer in image recognition, we propose a Multi-view Vision Transformer (MVT) for 3D object recognition. Since each patch feature in a Transformer block has a global reception field, it naturally achieves communications between patches from different views. Meanwhile, it takes much less inductive bias compared with its CNN counterparts. Considering both effectiveness and efficiency, we develop a global-local structure for our MVT. Our experiments on two public benchmarks, ModelNet40 and ModelNet10, demonstrate the competitive performance of our MVT.

@article{
 title = {Grounded Situation Recognition with Transformers},
 type = {article},
 year = {2021},
 websites = {https://github.com/jhcho99/gsrtr.},
 id = {e41e75a4-8222-377b-b429-a4645139c543},
 created = {2021-11-23T08:16:06.786Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:13.641Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Grounded Situation Recognition (GSR) is the task that not only classifies a salient action (verb), but also predicts entities (nouns) associated with semantic roles and their locations in the given image. Inspired by the remarkable success of Transformers in vision tasks, we propose a GSR model based on a Transformer encoder-decoder architecture. The attention mechanism of our model enables accurate verb classification by capturing high-level semantic feature of an image effectively, and allows the model to flexibly deal with the complicated and image-dependent relations between entities for improved noun classification and localization. Our model is the first Transformer architecture for GSR, and achieves the state of the art in every evaluation metric on the SWiG benchmark. Our code is available at https://github.com/jhcho99/gsrtr. Figure 1: Predictions of our model on the SWiG dataset.},
 bibtype = {article},
 author = {Cho, Junhyeong and Yoon, Youngseok and Lee, Hyeonjun and Kwak, Suha}
}

Grounded Situation Recognition (GSR) is the task that not only classifies a salient action (verb), but also predicts entities (nouns) associated with semantic roles and their locations in the given image. Inspired by the remarkable success of Transformers in vision tasks, we propose a GSR model based on a Transformer encoder-decoder architecture. The attention mechanism of our model enables accurate verb classification by capturing high-level semantic feature of an image effectively, and allows the model to flexibly deal with the complicated and image-dependent relations between entities for improved noun classification and localization. Our model is the first Transformer architecture for GSR, and achieves the state of the art in every evaluation metric on the SWiG benchmark. Our code is available at https://github.com/jhcho99/gsrtr. Figure 1: Predictions of our model on the SWiG dataset.

@article{
 title = {Exploiting Scene Depth for Object Detection with Multimodal Transformers},
 type = {article},
 year = {2021},
 id = {066f5d94-a1bd-3e87-b328-a86271e7156d},
 created = {2021-11-23T08:16:13.078Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:16.420Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {We propose a generic framework MEDUSA (Multimodal Estimated-Depth Unification with Self-Attention) to fuse RGB and depth information using multimodal transformers in the context of object detection. Unlike previous methods that use the depth measured from various physical sensors such as Kinect and Lidar, we show that the depth maps inferred by a monocular depth estimator can play an important role to enhance the performance of modern object detectors. In order to make use of the estimated depth, MEDUSA encompasses a robust feature extraction phase, followed by multimodal transformers for RGB-D fusion. The main strength of MEDUSA lies in its broad applicability for any existing large-scale RGB datasets including PASCAL VOC and Microsoft COCO. Extensive experiments with three datasets show that MEDUSA achieves higher precision than several strong baselines.},
 bibtype = {article},
 author = {Song, Hwanjun and Kim, Eunyoung and Jampani, Varun and Sun, Deqing and Lee, Jae-Gil and Yang, Ming-Hsuan}
}

We propose a generic framework MEDUSA (Multimodal Estimated-Depth Unification with Self-Attention) to fuse RGB and depth information using multimodal transformers in the context of object detection. Unlike previous methods that use the depth measured from various physical sensors such as Kinect and Lidar, we show that the depth maps inferred by a monocular depth estimator can play an important role to enhance the performance of modern object detectors. In order to make use of the estimated depth, MEDUSA encompasses a robust feature extraction phase, followed by multimodal transformers for RGB-D fusion. The main strength of MEDUSA lies in its broad applicability for any existing large-scale RGB datasets including PASCAL VOC and Microsoft COCO. Extensive experiments with three datasets show that MEDUSA achieves higher precision than several strong baselines.

@article{
 title = {SHI, MENG, XING, MA AND WATTENHOFER: 3D-RETR 3D-RETR: End-to-End Single and Multi-View 3D Reconstruction with Transformers},
 type = {article},
 year = {2021},
 websites = {https://github.com/FomalhautB/3D-RETR},
 id = {e2012a9e-c214-3b4c-be6c-fb38af734204},
 created = {2021-11-23T08:16:20.238Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:24.017Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {3D reconstruction aims to reconstruct 3D objects from 2D views. Previous works for 3D reconstruction mainly focus on feature matching between views or using CNNs as backbones. Recently, Transformers have been shown effective in multiple applications of computer vision. However, whether or not Transformers can be used for 3D reconstruction is still unclear. In this paper, we fill this gap by proposing 3D-RETR, which is able to perform end-to-end 3D REconstruction with TRansformers. 3D-RETR first uses a pretrained Transformer to extract visual features from 2D input images. 3D-RETR then uses another Transformer Decoder to obtain the voxel features. A CNN Decoder then takes as input the voxel features to obtain the reconstructed objects. 3D-RETR is capable of 3D reconstruction from a single view or multiple views. Experimental results on two datasets show that 3D-RETR reaches state-of-the-art performance on 3D reconstruction. Additional ablation study also demonstrates that 3D-DETR benefits from using Transformers.},
 bibtype = {article},
 author = {Shi, Zai and Meng, Zhao and Ch, Zhmeng@ethz and Xing, Yiran and Ma, Yunpu and Wattenhofer, Roger and Ch, Wattenhofer@ethz and Zurich, Eth and Aachen, Rwth}
}

3D reconstruction aims to reconstruct 3D objects from 2D views. Previous works for 3D reconstruction mainly focus on feature matching between views or using CNNs as backbones. Recently, Transformers have been shown effective in multiple applications of computer vision. However, whether or not Transformers can be used for 3D reconstruction is still unclear. In this paper, we fill this gap by proposing 3D-RETR, which is able to perform end-to-end 3D REconstruction with TRansformers. 3D-RETR first uses a pretrained Transformer to extract visual features from 2D input images. 3D-RETR then uses another Transformer Decoder to obtain the voxel features. A CNN Decoder then takes as input the voxel features to obtain the reconstructed objects. 3D-RETR is capable of 3D reconstruction from a single view or multiple views. Experimental results on two datasets show that 3D-RETR reaches state-of-the-art performance on 3D reconstruction. Additional ablation study also demonstrates that 3D-DETR benefits from using Transformers.

@article{
 title = {HAT-Net: A Hierarchical Transformer Graph Neural Network for Grading of Colorectal Cancer Histology Images},
 type = {article},
 year = {2021},
 id = {da613fa3-1037-382a-a51d-e19891adfbc1},
 created = {2021-11-23T08:16:26.731Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:32.957Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Graph-based learning methods have gained more attention in colorectal adenocarci-noma cancer (CRA) grading tasks for encoding the tissue structure information, which patch-wise CNN based methods fail to. Graph-based methods usually involve extracting nuclei features in the histology images as cell-graph node features and modeling the connections between nodes to construct cell-graphs. However, it is infeasible to directly train a classification model to extract nuclei features as we normally do in nature images since different types of nuclei often cluster together. We propose a Masked Nuclei Patch (MNP) approach to train a ResNet-50 as a strong feature encoder to extract more representative nuclei feature for enhancing the overall performance. Graph Neural Networks (GNNs) are often used to train cell-graphs for different tasks. But GNN may struggle to capture the long-range dependency due to its underlying recurrent structure. Therefore, we propose a new network architecture named HierArchical Transformer Graph Neural Network (HAT-Net), which merits both GNN and Transformer, as a strong competitor for CRA grading tasks. We have achieved the state-of-the-art results on two publicly available CRA grading datasets: the colorectal cancer (CRC) dataset (98.55%) and the extended colorectal cancer (Extended CRC) dataset (95.33%).},
 bibtype = {article},
 author = {Su, Yihan and Bai, Yu and Zhang, Bo and Zhang, Zheng and Wang, Wendong}
}

Graph-based learning methods have gained more attention in colorectal adenocarci-noma cancer (CRA) grading tasks for encoding the tissue structure information, which patch-wise CNN based methods fail to. Graph-based methods usually involve extracting nuclei features in the histology images as cell-graph node features and modeling the connections between nodes to construct cell-graphs. However, it is infeasible to directly train a classification model to extract nuclei features as we normally do in nature images since different types of nuclei often cluster together. We propose a Masked Nuclei Patch (MNP) approach to train a ResNet-50 as a strong feature encoder to extract more representative nuclei feature for enhancing the overall performance. Graph Neural Networks (GNNs) are often used to train cell-graphs for different tasks. But GNN may struggle to capture the long-range dependency due to its underlying recurrent structure. Therefore, we propose a new network architecture named HierArchical Transformer Graph Neural Network (HAT-Net), which merits both GNN and Transformer, as a strong competitor for CRA grading tasks. We have achieved the state-of-the-art results on two publicly available CRA grading datasets: the colorectal cancer (CRC) dataset (98.55%) and the extended colorectal cancer (Extended CRC) dataset (95.33%).

@article{
 title = {Feature Fusion Vision Transformer for Fine-Grained Visual Categorization},
 type = {article},
 year = {2021},
 id = {8f939260-038d-321d-91de-ea60e195f413},
 created = {2021-11-23T08:16:34.445Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:38.639Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {The core for tackling the fine-grained visual categorization (FGVC) is to learn subtle yet discriminative features. Most previous works achieve this by explicitly selecting the discriminative parts or integrating the attention mechanism via CNN-based approaches. However, these methods enhance the computational complexity and make the model dominated by the regions containing the most of the objects. Recently, vision transformer (ViT) has achieved SOTA performance on general image recognition tasks. The self-attention mechanism aggregates and weights the information from all patches to the classification token, making it perfectly suitable for FGVC. Nonetheless, the classification token in the deep layer pays more attention to the global information, lacking the local and low-level features that are essential for FGVC. In this work, we propose a novel pure transformer-based framework Feature Fusion Vision Transformer (FFVT) where we aggregate the important tokens from each transformer layer to compensate the local, low-level and middle-level information. We design a novel token selection module called mutual attention weight selection (MAWS) to guide the network effectively and efficiently towards selecting discriminative tokens without introducing extra parameters. We verify the effectiveness of FFVT on four benchmarks where FFVT achieves the state-of-the-art performance. Code is available at this link.},
 bibtype = {article},
 author = {Wang, Jun and Yu, Xiaohan and University, Griffith and Yongsheng Gao, Australia}
}

The core for tackling the fine-grained visual categorization (FGVC) is to learn subtle yet discriminative features. Most previous works achieve this by explicitly selecting the discriminative parts or integrating the attention mechanism via CNN-based approaches. However, these methods enhance the computational complexity and make the model dominated by the regions containing the most of the objects. Recently, vision transformer (ViT) has achieved SOTA performance on general image recognition tasks. The self-attention mechanism aggregates and weights the information from all patches to the classification token, making it perfectly suitable for FGVC. Nonetheless, the classification token in the deep layer pays more attention to the global information, lacking the local and low-level features that are essential for FGVC. In this work, we propose a novel pure transformer-based framework Feature Fusion Vision Transformer (FFVT) where we aggregate the important tokens from each transformer layer to compensate the local, low-level and middle-level information. We design a novel token selection module called mutual attention weight selection (MAWS) to guide the network effectively and efficiently towards selecting discriminative tokens without introducing extra parameters. We verify the effectiveness of FFVT on four benchmarks where FFVT achieves the state-of-the-art performance. Code is available at this link.

@article{
 title = {Localizing Objects with Self-Supervised Transformers and no Labels},
 type = {article},
 year = {2021},
 websites = {https://github.com/valeoai/LOST.},
 id = {6ed86187-4d6c-32b6-b58a-5a919dca068a},
 created = {2021-11-23T08:16:42.752Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:16:47.452Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Localizing objects in image collections without supervision can help to avoid expensive annotation campaigns. We propose a simple approach to this problem, that leverages the activation features of a vision transformer pre-trained in a self-supervised manner. Our method, LOST, does not require any external object proposal nor any exploration of the image collection; it operates on a single image. Yet, we outperform state-of-the-art object discovery methods by up to 8 CorLoc points on PASCAL VOC 2012. We also show that training a class-agnostic detector on the discovered objects boosts results by another 7 points. Moreover, we show promising results on the unsupervised object discovery task. The code can be found at https://github.com/valeoai/LOST. Figure 1: Three applications of LOST to unsupervised single-object discovery (left), multi-object discovery (middle) and object detection (right). In the latter case, objects discovered by LOST are clustered into categories, and cluster labels are used to train a classical object detector. Although large image collections are used to train the underlying image representation [13] and the detector [51], no annotation is ever used in the pipeline. See Figure 3 and Tables 1, 3 for more experiments.},
 bibtype = {article},
 author = {Siméoni, Oriane and Puy, Gilles and Vo, Huy V and Roburin, Simon and Gidaris, Spyros and Bursuc, Andrei and Pérez, Patrick and Marlet, Renaud and Ponce, Jean}
}

Localizing objects in image collections without supervision can help to avoid expensive annotation campaigns. We propose a simple approach to this problem, that leverages the activation features of a vision transformer pre-trained in a self-supervised manner. Our method, LOST, does not require any external object proposal nor any exploration of the image collection; it operates on a single image. Yet, we outperform state-of-the-art object discovery methods by up to 8 CorLoc points on PASCAL VOC 2012. We also show that training a class-agnostic detector on the discovered objects boosts results by another 7 points. Moreover, we show promising results on the unsupervised object discovery task. The code can be found at https://github.com/valeoai/LOST. Figure 1: Three applications of LOST to unsupervised single-object discovery (left), multi-object discovery (middle) and object detection (right). In the latter case, objects discovered by LOST are clustered into categories, and cluster labels are used to train a classical object detector. Although large image collections are used to train the underlying image representation [13] and the detector [51], no annotation is ever used in the pipeline. See Figure 3 and Tables 1, 3 for more experiments.

@article{
 title = {Image-Text Alignment using Adaptive Cross-attention with Transformer Encoder for Scene Graphs},
 type = {article},
 year = {2021},
 id = {6f8e91ad-49a8-38f8-86be-90375363f4f0},
 created = {2021-11-23T08:18:01.815Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:17.612Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Neural image and text encoders have been proposed to align the abstract image and symbolic text representation. Global-local and local-local information integration between two modalities are essential for an effective alignment. In this paper, we present RELation-aware Adaptive Cross-attention (RELAX) that achieves state-of-the-art performance in cross-modal retrieval tasks by incorporating several novel improvements. First, cross-attention methods integrate global-local information via weighted global feature of a modality (taken as value) for a local feature of the other modality (taken as query). We can make more accurate alignments if we could also consider the global weights of the query modality. To this end, we introduce adaptive embedding to consider the weights. Second, to enhance the usage of scene-graphs that can capture the high-level relation of local features, we introduce transformer encoders for textual scene graphs to align with visual scene graphs. Lastly, we use NT-XEnt loss that takes the weighted sum of the samples based on their importance. We show that our approach is effective in extensive experiments that outperform other state-of-the-art models.},
 bibtype = {article},
 author = {Song, Juyong and Choi, Sunghyun}
}

Neural image and text encoders have been proposed to align the abstract image and symbolic text representation. Global-local and local-local information integration between two modalities are essential for an effective alignment. In this paper, we present RELation-aware Adaptive Cross-attention (RELAX) that achieves state-of-the-art performance in cross-modal retrieval tasks by incorporating several novel improvements. First, cross-attention methods integrate global-local information via weighted global feature of a modality (taken as value) for a local feature of the other modality (taken as query). We can make more accurate alignments if we could also consider the global weights of the query modality. To this end, we introduce adaptive embedding to consider the weights. Second, to enhance the usage of scene-graphs that can capture the high-level relation of local features, we introduce transformer encoders for textual scene graphs to align with visual scene graphs. Lastly, we use NT-XEnt loss that takes the weighted sum of the samples based on their importance. We show that our approach is effective in extensive experiments that outperform other state-of-the-art models.

@article{
 title = {ASFormer: Transformer for Action Segmentation},
 type = {article},
 year = {2021},
 websites = {https://github.com/ChinaYi/ASFormer.},
 id = {32a5e47d-77c9-3f91-a32f-e2fcc98c25e2},
 created = {2021-11-23T08:18:08.907Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:14.752Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Algorithms for the action segmentation task typically use temporal models to predict what action is occurring at each frame for a minute-long daily activity. Recent studies have shown the potential of Transformer in modeling the relations among elements in sequential data. However, there are several major concerns when directly applying the Transformer to the action segmentation task, such as the lack of inductive biases with small training sets, the deficit in processing long input sequence, and the limitation of the decoder architecture to utilize temporal relations among multiple action segments to refine the initial predictions. To address these concerns, we design an efficient Transformer-based model for the action segmentation task, named ASFormer, with three distinctive characteristics: (i) We explicitly bring in the local connectivity inductive priors because of the high locality of features. It constrains the hypothesis space within a reliable scope, and is beneficial for the action segmentation task to learn a proper target function with small training sets. (ii) We apply a pre-defined hierarchical representation pattern that efficiently handles long input sequences. (iii) We carefully design the decoder to refine the initial predictions from the encoder. Extensive experiments on three public datasets demonstrate the effectiveness of our methods. Code is available at https://github.com/ChinaYi/ASFormer.},
 bibtype = {article},
 author = {Yi, Fangqiu and Wen, Hongyu and Jiang, Tingting}
}

Algorithms for the action segmentation task typically use temporal models to predict what action is occurring at each frame for a minute-long daily activity. Recent studies have shown the potential of Transformer in modeling the relations among elements in sequential data. However, there are several major concerns when directly applying the Transformer to the action segmentation task, such as the lack of inductive biases with small training sets, the deficit in processing long input sequence, and the limitation of the decoder architecture to utilize temporal relations among multiple action segments to refine the initial predictions. To address these concerns, we design an efficient Transformer-based model for the action segmentation task, named ASFormer, with three distinctive characteristics: (i) We explicitly bring in the local connectivity inductive priors because of the high locality of features. It constrains the hypothesis space within a reliable scope, and is beneficial for the action segmentation task to learn a proper target function with small training sets. (ii) We apply a pre-defined hierarchical representation pattern that efficiently handles long input sequences. (iii) We carefully design the decoder to refine the initial predictions from the encoder. Extensive experiments on three public datasets demonstrate the effectiveness of our methods. Code is available at https://github.com/ChinaYi/ASFormer.

@article{
 title = {Mitigating Bias in Visual Transformers via Targeted Alignment},
 type = {article},
 year = {2021},
 id = {92994ba8-fd1d-303c-8720-f785a7a6ed4f},
 created = {2021-11-23T08:18:11.707Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:14.677Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {As transformer architectures become increasingly prevalent in computer vision, it is critical to understand their fairness implications. We perform the first study of the fairness of transformers applied to computer vision and benchmark several bias miti-gation approaches from prior work. We visualize the feature space of the transformer self-attention modules and discover that a significant portion of the bias is encoded in the query matrix. With this knowledge, we propose TADeT, a targeted alignment strategy for debiasing transformers that aims to discover and remove bias primarily from query matrix features. We measure performance using Balanced Accuracy and Standard Accuracy , and fairness using Equalized Odds and Balanced Accuracy Difference. TADeT consistently leads to improved fairness over prior work on multiple attribute prediction tasks on the CelebA dataset, without compromising performance.},
 bibtype = {article},
 author = {Sudhakar, Sruthi and Prabhu, Viraj and Krishnakumar, Arvindkumar and Hoffman, Judy}
}

As transformer architectures become increasingly prevalent in computer vision, it is critical to understand their fairness implications. We perform the first study of the fairness of transformers applied to computer vision and benchmark several bias miti-gation approaches from prior work. We visualize the feature space of the transformer self-attention modules and discover that a significant portion of the bias is encoded in the query matrix. With this knowledge, we propose TADeT, a targeted alignment strategy for debiasing transformers that aims to discover and remove bias primarily from query matrix features. We measure performance using Balanced Accuracy and Standard Accuracy , and fairness using Equalized Odds and Balanced Accuracy Difference. TADeT consistently leads to improved fairness over prior work on multiple attribute prediction tasks on the CelebA dataset, without compromising performance.

@article{
 title = {Paying Attention to Varying Receptive Fields: Object Detection with Atrous Filters and Vision Transformers},
 type = {article},
 year = {2021},
 id = {93c8c6db-a45b-3ac2-9cab-946ffddf27d7},
 created = {2021-11-23T08:18:22.634Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:49.814Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Object detection represents a critical component in computer vision based on its unique ability to identify the location of one or more objects in an image or video. Given its importance, various approaches were proposed in an attempt to extract meaningful and representative features across different image scales. One such approach would be to vary the receptive fields during the feature extraction process. However, varying and adjusting the receptive field adds complexity to the process of scene understanding by introducing a higher degree of unimportant semantics into the feature maps. To solve this problem, we propose a novel object detection framework by unifying dilation modules (or atrous convolutions) with a vision transformer (DIL-ViT). The proposed model leverages atrous convolutions to generate rich multi-scale feature maps and employs a self-attention mechanism to enrich important backbone features. Specifically, the dila-tion (i.e., DIL) module enables feature fusions across varying scales from a single input feature map of specific scales. Through this method, we incorporate coarse semantics and fine details into the feature maps by convolving the features with different atrous rates in a multi-branch multi-level structure. By embedding DIL into various object detectors , we observe notable improvements in all of the compared evaluation metrics using the MS-COCO dataset. To further enhance the feature maps produced by the DIL, we then apply channel-wise attention using a vision transformer (i.e., ViT). Crucially, this approach removes unnecessary semantics present in the fused multi-scale feature map. Experimental results of DIL-ViT on the MS-COCO dataset exhibit substantial improvements in all of the compared evaluation metrics.},
 bibtype = {article},
 author = {Lam, Arthur and Lim, Junyi and Sutopo, Ricky and Monn Baskaran, Vishnu}
}

Object detection represents a critical component in computer vision based on its unique ability to identify the location of one or more objects in an image or video. Given its importance, various approaches were proposed in an attempt to extract meaningful and representative features across different image scales. One such approach would be to vary the receptive fields during the feature extraction process. However, varying and adjusting the receptive field adds complexity to the process of scene understanding by introducing a higher degree of unimportant semantics into the feature maps. To solve this problem, we propose a novel object detection framework by unifying dilation modules (or atrous convolutions) with a vision transformer (DIL-ViT). The proposed model leverages atrous convolutions to generate rich multi-scale feature maps and employs a self-attention mechanism to enrich important backbone features. Specifically, the dila-tion (i.e., DIL) module enables feature fusions across varying scales from a single input feature map of specific scales. Through this method, we incorporate coarse semantics and fine details into the feature maps by convolving the features with different atrous rates in a multi-branch multi-level structure. By embedding DIL into various object detectors , we observe notable improvements in all of the compared evaluation metrics using the MS-COCO dataset. To further enhance the feature maps produced by the DIL, we then apply channel-wise attention using a vision transformer (i.e., ViT). Crucially, this approach removes unnecessary semantics present in the fused multi-scale feature map. Experimental results of DIL-ViT on the MS-COCO dataset exhibit substantial improvements in all of the compared evaluation metrics.

@article{
 title = {Adversarial Robustness Comparison of Vision Transformer and MLP-Mixer to CNNs},
 type = {article},
 year = {2021},
 websites = {https://github.com/phibenz/robustness_comparison_},
 id = {db76e721-53a7-3911-91c8-fd817200a465},
 created = {2021-11-23T08:18:28.085Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:31.013Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model archi-tectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adver-sarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations. Code: https://github.com/phibenz/robustness_comparison_ vit_mlp-mixer_cnn.},
 bibtype = {article},
 author = {Benz, Philipp and Ham, Soomin and Zhang, Chaoning and Karjauv, Adil}
}

Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model archi-tectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adver-sarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations. Code: https://github.com/phibenz/robustness_comparison_ vit_mlp-mixer_cnn.

@article{
 title = {End-to-End Object Detection with Adaptive Clustering Transformer},
 type = {article},
 year = {2021},
 websites = {https://github.com/gaopengcuhk/SMCA-DETR/},
 id = {f9bdc9ef-4eef-3dbc-8298-ec993a1bd711},
 created = {2021-11-23T08:18:38.432Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:42.986Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {End-to-end Object Detection with Transformer (DETR) performs object detection with Transformer and achieves comparable performance with two-stage object detection like Faster-RCNN. However, DETR needs huge computational resources for training and inference due to the high-resolution spatial inputs. In this paper, a novel variant of transformer named Adaptive Clustering Transformer (ACT) has been proposed to reduce the computation cost for high-resolution input. ACT clusters the query features adaptively using Locality Sensitive Hashing (LSH) and approximates the query-key interaction using the prototype-key interaction. ACT can reduce the quadratic O(N 2) complexity inside self-attention into O(NK) where K is the number of prototypes in each layer. ACT can be a drop-in module replacing the original self-attention module without any training. ACT achieves a good balance between accuracy and computation cost (FLOPs). The code is available as supplementary for the ease of experiment replication and verification. Code is released at https://github.com/gaopengcuhk/SMCA-DETR/ tree/main/Adaptive_Cluster_Transformer},
 bibtype = {article},
 author = {Zheng, Minghang and Gao, Peng and Zhang, Renrui and Li, Kunchang and Wang, Xiaogang and Li, Hongsheng and Dong, Hao}
}

End-to-end Object Detection with Transformer (DETR) performs object detection with Transformer and achieves comparable performance with two-stage object detection like Faster-RCNN. However, DETR needs huge computational resources for training and inference due to the high-resolution spatial inputs. In this paper, a novel variant of transformer named Adaptive Clustering Transformer (ACT) has been proposed to reduce the computation cost for high-resolution input. ACT clusters the query features adaptively using Locality Sensitive Hashing (LSH) and approximates the query-key interaction using the prototype-key interaction. ACT can reduce the quadratic O(N 2) complexity inside self-attention into O(NK) where K is the number of prototypes in each layer. ACT can be a drop-in module replacing the original self-attention module without any training. ACT achieves a good balance between accuracy and computation cost (FLOPs). The code is available as supplementary for the ease of experiment replication and verification. Code is released at https://github.com/gaopengcuhk/SMCA-DETR/ tree/main/Adaptive_Cluster_Transformer

@article{
 title = {FETNet: Feature Exchange Transformer Network for RGB-D Object Detection},
 type = {article},
 year = {2021},
 id = {86ed9f88-ce64-3a47-b733-fd4ab1a06820},
 created = {2021-11-23T08:18:43.040Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:46.141Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {In RGB-D object detection, due to the inherent difference between the RGB and Depth modalities, it remains challenging to simultaneously leverage sensed photomet-ric and depth information. In this paper, to address this issue, we propose a Feature Exchange Transformer Network (FETNet), which consists of two well-designed components: the Feature Exchange Module (FEM), and the Multi-modal Vision Transformer (MViT). Specially, we propose the FEM to exchange part of the channels between RGB and depth features at each backbone stage, which facilitates the information flow, and bridges the gap, between the two modalities. Inspired by the success of Vision Transformer (ViT), we develop the variant MViT to effectively fuse multi-modal features and exploit the attention between the RGB and depth features. Different from previous methods developing from specified RGB detection algorithm, our proposal is generic. Extensive experiments prove that, when the proposed modules are integrated into mainstream RGB object detection methods, their RGB-D counterparts can obtain significant performance gains. Moreover, our FETNet surpasses state-of-the-art RGB-D detectors by 7.0% mAP on SUN RGB-D and 1.7% mAP on NYU Depth v2, which also well demonstrates the effectiveness of the proposed method.},
 bibtype = {article},
 author = {Xiao, Zhibin and Xue, Jing-Hao and Xie, Pengwei and Wang, Guijin}
}

In RGB-D object detection, due to the inherent difference between the RGB and Depth modalities, it remains challenging to simultaneously leverage sensed photomet-ric and depth information. In this paper, to address this issue, we propose a Feature Exchange Transformer Network (FETNet), which consists of two well-designed components: the Feature Exchange Module (FEM), and the Multi-modal Vision Transformer (MViT). Specially, we propose the FEM to exchange part of the channels between RGB and depth features at each backbone stage, which facilitates the information flow, and bridges the gap, between the two modalities. Inspired by the success of Vision Transformer (ViT), we develop the variant MViT to effectively fuse multi-modal features and exploit the attention between the RGB and depth features. Different from previous methods developing from specified RGB detection algorithm, our proposal is generic. Extensive experiments prove that, when the proposed modules are integrated into mainstream RGB object detection methods, their RGB-D counterparts can obtain significant performance gains. Moreover, our FETNet surpasses state-of-the-art RGB-D detectors by 7.0% mAP on SUN RGB-D and 1.7% mAP on NYU Depth v2, which also well demonstrates the effectiveness of the proposed method.

@article{
 title = {Multi-Exit Vision Transformer for Dynamic Inference},
 type = {article},
 year = {2021},
 websites = {https://gitlab.au.dk/maleci/multiexitvit.},
 id = {4a64a821-708f-397e-a9bf-5dae3a1e06c1},
 created = {2021-11-23T08:18:49.722Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:18:57.464Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Deep neural networks can be converted to multi-exit architectures by inserting early exit branches after some of their intermediate layers. This allows their inference process to become dynamic, which is useful for time critical IoT applications with stringent latency requirements, but with time-variant communication and computation resources. In particular, in edge computing systems and IoT networks where the exact computation time budget is variable and not known beforehand. Vision Transformer is a recently proposed architecture which has since found many applications across various domains of computer vision. In this work, we propose seven different architectures for early exit branches that can be used for dynamic inference in Vision Transformer backbones. Through extensive experiments involving both classification and regression problems, we show that each one of our proposed architectures could prove useful in the trade-off between accuracy and speed.},
 bibtype = {article},
 author = {Bakhtiarnia, Arian and Zhang, Qi and Iosifidis, Alexandros and Digit, Ai@ece Au Dk}
}

Deep neural networks can be converted to multi-exit architectures by inserting early exit branches after some of their intermediate layers. This allows their inference process to become dynamic, which is useful for time critical IoT applications with stringent latency requirements, but with time-variant communication and computation resources. In particular, in edge computing systems and IoT networks where the exact computation time budget is variable and not known beforehand. Vision Transformer is a recently proposed architecture which has since found many applications across various domains of computer vision. In this work, we propose seven different architectures for early exit branches that can be used for dynamic inference in Vision Transformer backbones. Through extensive experiments involving both classification and regression problems, we show that each one of our proposed architectures could prove useful in the trade-off between accuracy and speed.

@article{
 title = {Livestock Monitoring with Transformer},
 type = {article},
 year = {2021},
 websites = {http://www.fao.org/faostat/en/},
 id = {1f874559-3d83-3200-a662-f58847221269},
 created = {2021-11-23T08:19:02.987Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:10.853Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Tracking the behaviour of livestock enables early detection and thus prevention of contagious diseases in modern animal farms. Apart from economic gains, this would reduce the amount of antibiotics used in livestock farming which otherwise enters the human diet exasperating the epidemic of antibiotic resistance-a leading cause of death. We could use standard video cameras, available in most modern farms, to monitor livestock. However, most computer vision algorithms perform poorly on this task, primarily because, (i) animals bred in farms look identical, lacking any obvious spatial signature, (ii) none of the existing trackers are robust for long duration, and (iii) real-world conditions such as changing illumination, frequent occlusion, varying camera angles, and sizes of the animals make it hard for models to generalize. Given these challenges, we develop an end-to-end behaviour monitoring system for group-housed pigs to perform simultaneous instance level segmentation, tracking, action recognition and re-identification (STAR) tasks. We present STARFORMER, the first end-to-end multiple-object livestock monitoring framework that learns instance-level embeddings for grouped pigs through the use of transformer architecture. For benchmarking, we present PIGTRACE, a carefully curated dataset comprising video sequences with instance level bounding box, seg-mentation, tracking and activity classification of pigs in real indoor farming environment. Using simultaneous optimization on STAR tasks we show that STARFORMER outper-forms popular baseline models trained for individual tasks.},
 bibtype = {article},
 author = {Tangirala, Bhavesh and Bhandari, Ishan and Laszlo, Daniel and Gupta, Deepak K and Thomas, Rajat M and Arya, Devanshu and Nl, D Arya@uva}
}

Tracking the behaviour of livestock enables early detection and thus prevention of contagious diseases in modern animal farms. Apart from economic gains, this would reduce the amount of antibiotics used in livestock farming which otherwise enters the human diet exasperating the epidemic of antibiotic resistance-a leading cause of death. We could use standard video cameras, available in most modern farms, to monitor livestock. However, most computer vision algorithms perform poorly on this task, primarily because, (i) animals bred in farms look identical, lacking any obvious spatial signature, (ii) none of the existing trackers are robust for long duration, and (iii) real-world conditions such as changing illumination, frequent occlusion, varying camera angles, and sizes of the animals make it hard for models to generalize. Given these challenges, we develop an end-to-end behaviour monitoring system for group-housed pigs to perform simultaneous instance level segmentation, tracking, action recognition and re-identification (STAR) tasks. We present STARFORMER, the first end-to-end multiple-object livestock monitoring framework that learns instance-level embeddings for grouped pigs through the use of transformer architecture. For benchmarking, we present PIGTRACE, a carefully curated dataset comprising video sequences with instance level bounding box, seg-mentation, tracking and activity classification of pigs in real indoor farming environment. Using simultaneous optimization on STAR tasks we show that STARFORMER outper-forms popular baseline models trained for individual tasks.

@article{
 title = {Multi-Teacher Single-Student Visual Transformer with Multi-Level Attention for Face Spoofing Detection},
 type = {article},
 year = {2021},
 websites = {https://liveness.com/.},
 id = {9a2a9fb4-5e6d-3c67-afbd-896f43a5b8a8},
 created = {2021-11-23T08:19:07.581Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:10.417Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Face biometrics have attracted significant attention in many security-based applications. The presentation attack (PA) or face spoofing is a cybercriminal attempt to gain illegitimate access to a victim's device using photos, videos, or 3D artificial masks of the victim's face. Various deep learning approaches can tackle particular PA attacks when tested on standard datasets. However, these methods fail to generalize to complex environments or unseen datasets. We propose a new Multi-Teacher Single-Student (MTSS) visual Transformer with a multi-level attention design to improve the general-izability of face spoofing detection. Then, a novel Multi-Level Attention Module with a DropBlock (MAMD) is designed to strengthen discriminative features while dropping irrelevant spatial features to avoid overfitting. Finally, these rich convolutional feature sets are combined and fed into the MTSS network for face spoofing training. With this MAMD module, our method survives well under small training datasets with poorly lighted conditions. Experimental results demonstrate the superiority of our method when compared with several anti-spoofing methods on four datasets (CASIA-MFSD, Replay-Attack, MSU-MFSD, and OULU-NPU). Furthermore, our model can run on Jetson TX2 up to 80 FPS for real-world applications.},
 bibtype = {article},
 author = {Huang, Yao-Hui and Hsieh, Jun-Wei and Chang, Ming-Ching and Ke, Lipeng and Lyu, Siwei and Santra, Arpita Samanta}
}

Face biometrics have attracted significant attention in many security-based applications. The presentation attack (PA) or face spoofing is a cybercriminal attempt to gain illegitimate access to a victim's device using photos, videos, or 3D artificial masks of the victim's face. Various deep learning approaches can tackle particular PA attacks when tested on standard datasets. However, these methods fail to generalize to complex environments or unseen datasets. We propose a new Multi-Teacher Single-Student (MTSS) visual Transformer with a multi-level attention design to improve the general-izability of face spoofing detection. Then, a novel Multi-Level Attention Module with a DropBlock (MAMD) is designed to strengthen discriminative features while dropping irrelevant spatial features to avoid overfitting. Finally, these rich convolutional feature sets are combined and fed into the MTSS network for face spoofing training. With this MAMD module, our method survives well under small training datasets with poorly lighted conditions. Experimental results demonstrate the superiority of our method when compared with several anti-spoofing methods on four datasets (CASIA-MFSD, Replay-Attack, MSU-MFSD, and OULU-NPU). Furthermore, our model can run on Jetson TX2 up to 80 FPS for real-world applications.

@article{
 title = {OODformer: Out-Of-Distribution Detection Transformer},
 type = {article},
 year = {2021},
 websites = {https://github.com/rajatkoner08/oodformer.},
 id = {59a23fd0-d1f9-32e5-944f-67758b105b39},
 created = {2021-11-23T08:19:13.339Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:16.779Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {A serious problem in image classification is that a trained model might perform well for input data that originates from the same distribution as the data available for model training, but performs much worse for out-of-distribution (OOD) samples. In real-world safety-critical applications, in particular, it is important to be aware if a new data point is OOD. To date, OOD detection is typically addressed using either confidence scores, auto-encoder based reconstruction, or contrastive learning. However, the global image context has not yet been explored to discriminate the non-local objectness between in-distribution and OOD samples. This paper proposes a first-of-its-kind OOD detection architecture named OODformer that leverages the contextualization capabilities of the transformer. Incorporating the transformer as the principal feature extractor allows us to exploit the object concepts and their discriminatory attributes along with their co-occurrence via visual attention. Based on contextualised embedding, we demonstrate OOD detection using both class-conditioned latent space similarity and a network confidence score. Our approach shows improved generalizability across various datasets. We have achieved a new state-of-the-art result on CIFAR-10/-100 and ImageNet30. Code is available at : https://github.com/rajatkoner08/oodformer.},
 bibtype = {article},
 author = {Koner, Rajat and Sinhamahapatra, Poulami and Roscher, Karsten and Günnemann, Stephan and Tresp, Volker}
}

A serious problem in image classification is that a trained model might perform well for input data that originates from the same distribution as the data available for model training, but performs much worse for out-of-distribution (OOD) samples. In real-world safety-critical applications, in particular, it is important to be aware if a new data point is OOD. To date, OOD detection is typically addressed using either confidence scores, auto-encoder based reconstruction, or contrastive learning. However, the global image context has not yet been explored to discriminate the non-local objectness between in-distribution and OOD samples. This paper proposes a first-of-its-kind OOD detection architecture named OODformer that leverages the contextualization capabilities of the transformer. Incorporating the transformer as the principal feature extractor allows us to exploit the object concepts and their discriminatory attributes along with their co-occurrence via visual attention. Based on contextualised embedding, we demonstrate OOD detection using both class-conditioned latent space similarity and a network confidence score. Our approach shows improved generalizability across various datasets. We have achieved a new state-of-the-art result on CIFAR-10/-100 and ImageNet30. Code is available at : https://github.com/rajatkoner08/oodformer.

@article{
 title = {Transformer-based Monocular Depth Estimation with Attention Supervision},
 type = {article},
 year = {2021},
 websites = {https://github.com/WJ-Chang-42/ASTransformer.},
 id = {86494662-a547-383f-b7c0-8a86cd76d42f},
 created = {2021-11-23T08:19:19.453Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:24.846Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Transformer, which excels in capturing long-range dependencies, has shown great performance in a variety of computer vision tasks. In this paper, we propose a hybrid network with a Transformer-based encoder and a CNN-based decoder for monocular depth estimation. The encoder follows the architecture of classical Vision Transformer. To better exploit the potential of the Transformer encoder, we introduce the Attention Supervision to the Transformer layer, which enhances the representative ability. The down-sampling operations before the Transformer encoder lead to degradation of the details in the predicted depth map. Thus, we devise an Attention-based Up-sample Block and deploy it to compensate the texture features. Experiments on both indoor and outdoor datasets demonstrate that the proposed method achieves the state-of-the-art performance on both quantitative and qualitative evaluations. The source code and trained models can be downloaded at https://github.com/WJ-Chang-42/ASTransformer.},
 bibtype = {article},
 author = {Chang, Wenjie and Zhang, Yueyi and Xiong, Zhiwei}
}

Transformer, which excels in capturing long-range dependencies, has shown great performance in a variety of computer vision tasks. In this paper, we propose a hybrid network with a Transformer-based encoder and a CNN-based decoder for monocular depth estimation. The encoder follows the architecture of classical Vision Transformer. To better exploit the potential of the Transformer encoder, we introduce the Attention Supervision to the Transformer layer, which enhances the representative ability. The down-sampling operations before the Transformer encoder lead to degradation of the details in the predicted depth map. Thus, we devise an Attention-based Up-sample Block and deploy it to compensate the texture features. Experiments on both indoor and outdoor datasets demonstrate that the proposed method achieves the state-of-the-art performance on both quantitative and qualitative evaluations. The source code and trained models can be downloaded at https://github.com/WJ-Chang-42/ASTransformer.

@article{
 title = {Few-Shot Temporal Action Localization with Query Adaptive Transformer},
 type = {article},
 year = {2021},
 websites = {https://github.com/sauradip/fewshotQAT},
 id = {bf359bae-3b9d-3d2a-9ffd-1e14b65e549f},
 created = {2021-11-23T08:19:31.278Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:34.109Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Existing temporal action localization (TAL) works rely on a large number of training videos with exhaustive segment-level annotation, preventing them from scaling to new classes. As a solution to this problem, few-shot TAL (FS-TAL) aims to adapt a model to a new class represented by as few as a single video. Exiting FS-TAL methods assume trimmed training videos for new classes. However, this setting is not only unnatural-actions are typically captured in untrimmed videos, but also ignores background video segments containing vital contextual cues for foreground action segmentation. In this work, we first propose a new FS-TAL setting by proposing to use untrimmed training videos. Further, a novel FS-TAL model is proposed which maximizes the knowledge transfer from training classes whilst enabling the model to be dynamically adapted to both the new class and each video of that class simultaneously. This is achieved by introducing a query adaptive Transformer in the model. Extensive experiments on two action localization benchmarks demonstrate that our method can outperform all the state-of-the-art alternatives significantly in both single-domain and cross-domain scenarios. The source code can be found in https://github.com/sauradip/fewshotQAT},
 bibtype = {article},
 author = {Nag, Sauradip and Zhu, Xiatian and Xiang, Tao}
}

Existing temporal action localization (TAL) works rely on a large number of training videos with exhaustive segment-level annotation, preventing them from scaling to new classes. As a solution to this problem, few-shot TAL (FS-TAL) aims to adapt a model to a new class represented by as few as a single video. Exiting FS-TAL methods assume trimmed training videos for new classes. However, this setting is not only unnatural-actions are typically captured in untrimmed videos, but also ignores background video segments containing vital contextual cues for foreground action segmentation. In this work, we first propose a new FS-TAL setting by proposing to use untrimmed training videos. Further, a novel FS-TAL model is proposed which maximizes the knowledge transfer from training classes whilst enabling the model to be dynamically adapted to both the new class and each video of that class simultaneously. This is achieved by introducing a query adaptive Transformer in the model. Extensive experiments on two action localization benchmarks demonstrate that our method can outperform all the state-of-the-art alternatives significantly in both single-domain and cross-domain scenarios. The source code can be found in https://github.com/sauradip/fewshotQAT

@article{
 title = {PROF: SATOSHI IKEHATA PS-Transformer: Learning Sparse Photometric Stereo Network using Self-Attention Mechanism Satoshi Ikehata},
 type = {article},
 year = {2021},
 websites = {https://satoshi-ikehata.github.io/},
 id = {6ce4ec11-c7a9-39eb-af39-f75ff339dc96},
 created = {2021-11-23T08:19:36.456Z},
 accessed = {2021-11-23},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-11-23T08:19:39.730Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {0facbd6c-68b8-4efe-af4f-9311071d6b5c},
 private_publication = {false},
 abstract = {Existing deep calibrated photometric stereo networks basically aggregate observations under different lights based on the pre-defined operations such as linear projection and max pooling. While they are effective with the dense capture, simple first-order operations often fail to capture the high-order interactions among observations under small number of different lights. To tackle this issue, this paper presents a deep sparse calibrated photometric stereo network named PS-Transformer which leverages the learnable self-attention mechanism to properly capture the complex inter-image interactions. PS-Transformer builds upon the dual-branch design to explore both pixel-wise and image-wise features and individual feature is trained with the intermediate surface normal supervision to maximize geometric feasibility. A new synthetic dataset named CyclesPS+ is also presented with the comprehensive analysis to successfully train the photometric stereo networks. Extensive results on the publicly available benchmark datasets demonstrate that the surface normal prediction accuracy of the proposed method significantly outperforms other state-of-the-art algorithms with the same number of input images and is even comparable to that of dense algorithms which input 10× larger number of images.},
 bibtype = {article},
 author = {}
}

Existing deep calibrated photometric stereo networks basically aggregate observations under different lights based on the pre-defined operations such as linear projection and max pooling. While they are effective with the dense capture, simple first-order operations often fail to capture the high-order interactions among observations under small number of different lights. To tackle this issue, this paper presents a deep sparse calibrated photometric stereo network named PS-Transformer which leverages the learnable self-attention mechanism to properly capture the complex inter-image interactions. PS-Transformer builds upon the dual-branch design to explore both pixel-wise and image-wise features and individual feature is trained with the intermediate surface normal supervision to maximize geometric feasibility. A new synthetic dataset named CyclesPS+ is also presented with the comprehensive analysis to successfully train the photometric stereo networks. Extensive results on the publicly available benchmark datasets demonstrate that the surface normal prediction accuracy of the proposed method significantly outperforms other state-of-the-art algorithms with the same number of input images and is even comparable to that of dense algorithms which input 10× larger number of images.

@article{
 title = {Training Graph Neural Networks with 1000 Layers},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2106.07476},
 id = {d649ab2e-9617-3883-b3db-9fe0e1d2922c},
 created = {2021-12-09T14:47:57.163Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-12-09T14:48:02.742Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {13f2c27e-5827-43b2-8a2b-d62c62bc0ecc},
 private_publication = {false},
 abstract = {Deep graph neural networks (GNNs) have achieved excellent results on various tasks on increasingly large graph datasets with millions of nodes and edges. However, memory complexity has become a major obstacle when training deep GNNs for practical applications due to the immense number of nodes, edges, and intermediate activations. To improve the scalability of GNNs, prior works propose smart graph sampling or partitioning strategies to train GNNs with a smaller set of nodes or sub-graphs. In this work, we study reversible connections, group convolutions, weight tying, and equilibrium models to advance the memory and parameter efficiency of GNNs. We find that reversible connections in combination with deep network architectures enable the training of overparameterized GNNs that significantly outperform existing methods on multiple datasets. Our models RevGNN-Deep (1001 layers with 80 channels each) and RevGNN-Wide (448 layers with 224 channels each) were both trained on a single commodity GPU and achieve an ROC-AUC of $87.74 \pm 0.13$ and $88.24 \pm 0.15$ on the ogbn-proteins dataset. To the best of our knowledge, RevGNN-Deep is the deepest GNN in the literature by one order of magnitude. Please visit our project website https://www.deepgcns.org/arch/gnn1000 for more information.},
 bibtype = {article},
 author = {Li, Guohao and Müller, Matthias and Ghanem, Bernard and Koltun, Vladlen}
}

Deep graph neural networks (GNNs) have achieved excellent results on various tasks on increasingly large graph datasets with millions of nodes and edges. However, memory complexity has become a major obstacle when training deep GNNs for practical applications due to the immense number of nodes, edges, and intermediate activations. To improve the scalability of GNNs, prior works propose smart graph sampling or partitioning strategies to train GNNs with a smaller set of nodes or sub-graphs. In this work, we study reversible connections, group convolutions, weight tying, and equilibrium models to advance the memory and parameter efficiency of GNNs. We find that reversible connections in combination with deep network architectures enable the training of overparameterized GNNs that significantly outperform existing methods on multiple datasets. Our models RevGNN-Deep (1001 layers with 80 channels each) and RevGNN-Wide (448 layers with 224 channels each) were both trained on a single commodity GPU and achieve an ROC-AUC of $87.74 \pm 0.13$ and $88.24 \pm 0.15$ on the ogbn-proteins dataset. To the best of our knowledge, RevGNN-Deep is the deepest GNN in the literature by one order of magnitude. Please visit our project website https://www.deepgcns.org/arch/gnn1000 for more information.

@article{
 title = {Adversarial Attack on Graph Neural Networks as An Influence Maximization Problem},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2106.10785},
 id = {493b5d2d-9763-3326-94c5-faacbc88615a},
 created = {2022-01-05T09:23:15.556Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:26.889Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph neural networks (GNNs) have attracted increasing interests. With broad deployments of GNNs in real-world applications, there is an urgent need for understanding the robustness of GNNs under adversarial attacks, especially in realistic setups. In this work, we study the problem of attacking GNNs in a restricted and realistic setup, by perturbing the features of a small set of nodes, with no access to model parameters and model predictions. Our formal analysis draws a connection between this type of attacks and an influence maximization problem on the graph. This connection not only enhances our understanding on the problem of adversarial attack on GNNs, but also allows us to propose a group of effective and practical attack strategies. Our experiments verify that the proposed attack strategies significantly degrade the performance of three popular GNN models and outperform baseline adversarial attack strategies.},
 bibtype = {article},
 author = {Ma, Jiaqi and Deng, Junwei and Mei, Qiaozhu}
}

Graph neural networks (GNNs) have attracted increasing interests. With broad deployments of GNNs in real-world applications, there is an urgent need for understanding the robustness of GNNs under adversarial attacks, especially in realistic setups. In this work, we study the problem of attacking GNNs in a restricted and realistic setup, by perturbing the features of a small set of nodes, with no access to model parameters and model predictions. Our formal analysis draws a connection between this type of attacks and an influence maximization problem on the graph. This connection not only enhances our understanding on the problem of adversarial attack on GNNs, but also allows us to propose a group of effective and practical attack strategies. Our experiments verify that the proposed attack strategies significantly degrade the performance of three popular GNN models and outperform baseline adversarial attack strategies.

@article{
 title = {Adversarial Attacks on Graph Classification via Bayesian Optimisation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.02842},
 id = {e90f4522-1c23-3ff2-bcc1-17e9e770699e},
 created = {2022-01-05T09:23:15.561Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:11.812Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph neural networks, a popular class of models effective in a wide range of graph-based learning tasks, have been shown to be vulnerable to adversarial attacks. While the majority of the literature focuses on such vulnerability in node-level classification tasks, little effort has been dedicated to analysing adversarial attacks on graph-level classification, an important problem with numerous real-life applications such as biochemistry and social network analysis. The few existing methods often require unrealistic setups, such as access to internal information of the victim models, or an impractically-large number of queries. We present a novel Bayesian optimisation-based attack method for graph classification models. Our method is black-box, query-efficient and parsimonious with respect to the perturbation applied. We empirically validate the effectiveness and flexibility of the proposed method on a wide range of graph classification tasks involving varying graph properties, constraints and modes of attack. Finally, we analyse common interpretable patterns behind the adversarial samples produced, which may shed further light on the adversarial robustness of graph classification models.},
 bibtype = {article},
 author = {Wan, Xingchen and Kenlay, Henry and Ru, Binxin and Blaas, Arno and Osborne, Michael A. and Dong, Xiaowen},
 number = {NeurIPS}
}

Graph neural networks, a popular class of models effective in a wide range of graph-based learning tasks, have been shown to be vulnerable to adversarial attacks. While the majority of the literature focuses on such vulnerability in node-level classification tasks, little effort has been dedicated to analysing adversarial attacks on graph-level classification, an important problem with numerous real-life applications such as biochemistry and social network analysis. The few existing methods often require unrealistic setups, such as access to internal information of the victim models, or an impractically-large number of queries. We present a novel Bayesian optimisation-based attack method for graph classification models. Our method is black-box, query-efficient and parsimonious with respect to the perturbation applied. We empirically validate the effectiveness and flexibility of the proposed method on a wide range of graph classification tasks involving varying graph properties, constraints and modes of attack. Finally, we analyse common interpretable patterns behind the adversarial samples produced, which may shed further light on the adversarial robustness of graph classification models.

@article{
 title = {Robust graph convolutional networks with directional graph adversarial training},
 type = {article},
 year = {2021},
 keywords = {Adversarial training,Graph adversarial learning,Graph convolutional networks,Robustness},
 pages = {7812-7826},
 volume = {51},
 publisher = {Applied Intelligence},
 id = {37c0232c-275e-35cc-b492-108d13816464},
 created = {2022-01-05T09:23:15.677Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:29.056Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph convolutional networks (GCNs), an emerging type of neural network model on graphs, have presented state-of-the-art performance on the node classification task. However, recent studies show that neural networks are vulnerable to the small but deliberate perturbations on input features. And GCNs could be more sensitive to the perturbations since the perturbations from neighbor nodes exacerbate the impact on a target node through the convolution. Adversarial training (AT) is a regularization technique that has been shown capable of improving the robustness of the model against perturbations on image classification. However, directly adopting AT on GCNs is less effective since AT regards examples as independent of each other and does not consider the impact from connected examples. In this work, we explore AT on graph and propose a graph-specific AT method, Directional Graph Adversarial Training (DGAT), which incorporates the graph structure into the adversarial process and automatically identifies the impact of perturbations from neighbor nodes. Concretely, we consider the impact from the connected nodes to define the neighbor perturbation which restricts the perturbation direction on node features towards their neighbor nodes, and additionally introduce an adversarial regularizer to defend the worst-case perturbations. In this way, DGAT can resist the impact of worst-case adversarial perturbations and reduce the impact of perturbations from neighbor nodes. Extensive experiments demonstrate that DGAT can effectively improve the robustness and generalization performance of GCNs. Specially, GCNs with DGAT can provide better performance when there are rare few labels available for training.},
 bibtype = {article},
 author = {Hu, Weibo and Chen, Chuan and Chang, Yaomin and Zheng, Zibin and Du, Yunfei},
 doi = {10.1007/s10489-021-02272-y},
 journal = {Applied Intelligence},
 number = {11}
}

Graph convolutional networks (GCNs), an emerging type of neural network model on graphs, have presented state-of-the-art performance on the node classification task. However, recent studies show that neural networks are vulnerable to the small but deliberate perturbations on input features. And GCNs could be more sensitive to the perturbations since the perturbations from neighbor nodes exacerbate the impact on a target node through the convolution. Adversarial training (AT) is a regularization technique that has been shown capable of improving the robustness of the model against perturbations on image classification. However, directly adopting AT on GCNs is less effective since AT regards examples as independent of each other and does not consider the impact from connected examples. In this work, we explore AT on graph and propose a graph-specific AT method, Directional Graph Adversarial Training (DGAT), which incorporates the graph structure into the adversarial process and automatically identifies the impact of perturbations from neighbor nodes. Concretely, we consider the impact from the connected nodes to define the neighbor perturbation which restricts the perturbation direction on node features towards their neighbor nodes, and additionally introduce an adversarial regularizer to defend the worst-case perturbations. In this way, DGAT can resist the impact of worst-case adversarial perturbations and reduce the impact of perturbations from neighbor nodes. Extensive experiments demonstrate that DGAT can effectively improve the robustness and generalization performance of GCNs. Specially, GCNs with DGAT can provide better performance when there are rare few labels available for training.

@article{
 title = {Graph Adversarial Attack via Rewiring},
 type = {article},
 year = {2021},
 keywords = {adversarial attack,graph neural networks,rewiring},
 pages = {1161-1169},
 id = {9d034f24-6a22-35cc-8097-58813eddcaff},
 created = {2022-01-05T09:23:15.687Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:23.073Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph Neural Networks (GNNs) have demonstrated their powerful capability in learning representations for graph-structured data. Consequently, they have enhanced the performance of many graph-related tasks such as node classification and graph classification. However, it is evident from recent studies that GNNs are vulnerable to adversarial attacks. Their performance can be largely impaired by deliberately adding carefully created unnoticeable perturbations to the graph. Existing attacking methods often produce perturbation by adding/deleting a few edges, which might be noticeable even when the number of modified edges is small. In this paper, we propose a graph rewiring operation to perform the attack. It can affect the graph in a less noticeable way compared to existing operations such as adding/deleting edges. We then utilize deep reinforcement learning to learn the strategy to effectively perform the rewiring operations. Experiments on real-world graphs demonstrate the effectiveness of the proposed framework. To understand the proposed framework, we further analyze how its generated perturbation impacts the target model and the advantages of the rewiring operations. The implementation of the proposed framework is available at https://github.com/alge24/ReWatt.},
 bibtype = {article},
 author = {Ma, Yao and Wang, Suhang and Derr, Tyler and Wu, Lingfei and Tang, Jiliang},
 doi = {10.1145/3447548.3467416},
 journal = {Proceedings of the ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}
}

Graph Neural Networks (GNNs) have demonstrated their powerful capability in learning representations for graph-structured data. Consequently, they have enhanced the performance of many graph-related tasks such as node classification and graph classification. However, it is evident from recent studies that GNNs are vulnerable to adversarial attacks. Their performance can be largely impaired by deliberately adding carefully created unnoticeable perturbations to the graph. Existing attacking methods often produce perturbation by adding/deleting a few edges, which might be noticeable even when the number of modified edges is small. In this paper, we propose a graph rewiring operation to perform the attack. It can affect the graph in a less noticeable way compared to existing operations such as adding/deleting edges. We then utilize deep reinforcement learning to learn the strategy to effectively perform the rewiring operations. Experiments on real-world graphs demonstrate the effectiveness of the proposed framework. To understand the proposed framework, we further analyze how its generated perturbation impacts the target model and the advantages of the rewiring operations. The implementation of the proposed framework is available at https://github.com/alge24/ReWatt.

@article{
 title = {Task and Model Agnostic Adversarial Attack on Graph Neural Networks},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2112.13267},
 id = {13ce2963-423a-3037-b682-f167861bdfd4},
 created = {2022-01-05T09:23:15.689Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:24.258Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph neural networks (GNNs) have witnessed significant adoption in the industry owing to impressive performance on various predictive tasks. Performance alone, however, is not enough. Any widely deployed machine learning algorithm must be robust to adversarial attacks. In this work, we investigate this aspect for GNNs, identify vulnerabilities, and link them to graph properties that may potentially lead to the development of more secure and robust GNNs. Specifically, we formulate the problem of task and model agnostic evasion attacks where adversaries modify the test graph to affect the performance of any unknown downstream task. The proposed algorithm, GRAND ($Gr$aph $A$ttack via $N$eighborhood $D$istortion) shows that distortion of node neighborhoods is effective in drastically compromising prediction performance. Although neighborhood distortion is an NP-hard problem, GRAND designs an effective heuristic through a novel combination of Graph Isomorphism Network with deep $Q$-learning. Extensive experiments on real datasets show that, on average, GRAND is up to $50\%$ more effective than state of the art techniques, while being more than $100$ times faster.},
 bibtype = {article},
 author = {Sharma, Kartik and Verma, Samidha and Medya, Sourav and Ranu, Sayan and Bhattacharya, Arnab}
}

Graph neural networks (GNNs) have witnessed significant adoption in the industry owing to impressive performance on various predictive tasks. Performance alone, however, is not enough. Any widely deployed machine learning algorithm must be robust to adversarial attacks. In this work, we investigate this aspect for GNNs, identify vulnerabilities, and link them to graph properties that may potentially lead to the development of more secure and robust GNNs. Specifically, we formulate the problem of task and model agnostic evasion attacks where adversaries modify the test graph to affect the performance of any unknown downstream task. The proposed algorithm, GRAND ($Gr$aph $A$ttack via $N$eighborhood $D$istortion) shows that distortion of node neighborhoods is effective in drastically compromising prediction performance. Although neighborhood distortion is an NP-hard problem, GRAND designs an effective heuristic through a novel combination of Graph Isomorphism Network with deep $Q$-learning. Extensive experiments on real datasets show that, on average, GRAND is up to $50\%$ more effective than state of the art techniques, while being more than $100$ times faster.

@article{
 title = {Understanding Structural Vulnerability in Graph Convolutional Networks},
 type = {article},
 year = {2021},
 pages = {2249-2255},
 id = {9bdc1b50-e4f7-3d41-9360-c2873c4c7f9e},
 created = {2022-01-05T09:23:15.839Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:32.747Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Recent studies have shown that Graph Convolutional Networks (GCNs) are vulnerable to adversarial attacks on the graph structure. Although multiple works have been proposed to improve their robustness against such structural adversarial attacks, the reasons for the success of the attacks remain unclear. In this work, we theoretically and empirically demonstrate that structural adversarial examples can be attributed to the non-robust aggregation scheme (i.e., the weighted mean) of GCNs. Specifically, our analysis takes advantage of the breakdown point which can quantitatively measure the robustness of aggregation schemes. The key insight is that weighted mean, as the basic design of GCNs, has a low breakdown point and its output can be dramatically changed by injecting a single edge. We show that adopting the aggregation scheme with a high breakdown point (e.g., median or trimmed mean) could significantly enhance the robustness of GCNs against structural attacks. Extensive experiments on four real-world datasets demonstrate that such a simple but effective method achieves the best robustness performance compared to state-of-the-art models.},
 bibtype = {article},
 author = {Chen, Liang and Li, Jintang and Peng, Qibiao and Liu, Yang and Zheng, Zibin and Yang, Carl},
 doi = {10.24963/ijcai.2021/310}
}

Recent studies have shown that Graph Convolutional Networks (GCNs) are vulnerable to adversarial attacks on the graph structure. Although multiple works have been proposed to improve their robustness against such structural adversarial attacks, the reasons for the success of the attacks remain unclear. In this work, we theoretically and empirically demonstrate that structural adversarial examples can be attributed to the non-robust aggregation scheme (i.e., the weighted mean) of GCNs. Specifically, our analysis takes advantage of the breakdown point which can quantitatively measure the robustness of aggregation schemes. The key insight is that weighted mean, as the basic design of GCNs, has a low breakdown point and its output can be dramatically changed by injecting a single edge. We show that adopting the aggregation scheme with a high breakdown point (e.g., median or trimmed mean) could significantly enhance the robustness of GCNs against structural attacks. Extensive experiments on four real-world datasets demonstrate that such a simple but effective method achieves the best robustness performance compared to state-of-the-art models.

@article{
 title = {Graph Universal Adversarial Attacks: A Few Bad Actors Ruin Graph Learning Models},
 type = {article},
 year = {2021},
 pages = {3328-3334},
 id = {db81a873-a406-3b9b-bfdd-7c76bbcd2d04},
 created = {2022-01-05T09:23:16.325Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:23:57.768Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Deep neural networks, while generalize well, are known to be sensitive to small adversarial perturbations. This phenomenon poses severe security threat and calls for in-depth investigation of the robustness of deep learning models. With the emergence of neural networks for graph structured data, similar investigations are urged to understand their robustness. It has been found that adversarially perturbing the graph structure and/or node features may result in a significant degradation of the model performance. In this work, we show from a different angle that such fragility similarly occurs if the graph contains a few bad-actor nodes, which compromise a trained graph neural network through flipping the connections to any targeted victim. Worse, the bad actors found for one graph model severely compromise other models as well. We call the bad actors ``anchor nodes'' and propose an algorithm, named GUA, to identify them. Thorough empirical investigations suggest an interesting finding that the anchor nodes often belong to the same class; and they also corroborate the intuitive trade-off between the number of anchor nodes and the attack success rate. For the dataset Cora which contains 2708 nodes, as few as six anchor nodes will result in an attack success rate higher than 80% for GCN and other three models.},
 bibtype = {article},
 author = {Zang, Xiao and Xie, Yi and Chen, Jie and Yuan, Bo},
 doi = {10.24963/ijcai.2021/458}
}

Deep neural networks, while generalize well, are known to be sensitive to small adversarial perturbations. This phenomenon poses severe security threat and calls for in-depth investigation of the robustness of deep learning models. With the emergence of neural networks for graph structured data, similar investigations are urged to understand their robustness. It has been found that adversarially perturbing the graph structure and/or node features may result in a significant degradation of the model performance. In this work, we show from a different angle that such fragility similarly occurs if the graph contains a few bad-actor nodes, which compromise a trained graph neural network through flipping the connections to any targeted victim. Worse, the bad actors found for one graph model severely compromise other models as well. We call the bad actors ``anchor nodes'' and propose an algorithm, named GUA, to identify them. Thorough empirical investigations suggest an interesting finding that the anchor nodes often belong to the same class; and they also corroborate the intuitive trade-off between the number of anchor nodes and the attack success rate. For the dataset Cora which contains 2708 nodes, as few as six anchor nodes will result in an attack success rate higher than 80% for GCN and other three models.

@article{
 title = {Adversarial Attacks and Defenses on Graphs},
 type = {article},
 year = {2021},
 pages = {19-34},
 volume = {22},
 id = {d75d4e71-09a2-3d8c-b8a3-a138ba09791a},
 created = {2022-01-05T09:23:16.452Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:23:51.812Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Deep neural networks (DNNs) have achieved significant performance in various tasks. However, recent studies have shown that DNNs can be easily fooled by small perturbation on the input, called adversarial attacks.},
 bibtype = {article},
 author = {Jin, Wei and Li, Yaxing and Xu, Han and Wang, Yiqi and Ji, Shuiwang and Aggarwal, Charu and Tang, Jiliang},
 doi = {10.1145/3447556.3447566},
 journal = {ACM SIGKDD Explorations Newsletter},
 number = {2}
}

Deep neural networks (DNNs) have achieved significant performance in various tasks. However, recent studies have shown that DNNs can be easily fooled by small perturbation on the input, called adversarial attacks.

@article{
 title = {Detection and Defense of Topological Adversarial Attacks on Graphs},
 type = {article},
 year = {2021},
 pages = {2989-2997},
 volume = {130},
 websites = {http://proceedings.mlr.press/v130/zhang21i.html%0Ahttp://proceedings.mlr.press/v130/zhang21i/zhang21i.pdf},
 id = {5cc56d4e-a026-36ac-bfc6-4363f81c109d},
 created = {2022-01-05T09:23:16.472Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:02.878Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph neural network (GNN) models achieve superior performance when classifying nodes in graph-structured data. Given that state-of-the-art GNNs share many similarities with their CNN cousins and that CNNs suffer ad-versarial vulnerabilities, there has also been interest in exploring analogous vulnerabilities in GNNs. Indeed, recent work has demonstrated that node classification performance of several graph models, including the popular graph convolution network (GCN) model, can be severely degraded through adversar-ial perturbations to the graph structure and the node features. In this work, we take a first step towards detecting adversarial attacks against graph models. We first propose a straightforward single node threshold test for detecting nodes subject to targeted attacks. Subsequently, we describe a kernel-based two-sample test for detecting whether a given subset of nodes within a graph has been maliciously corrupted. The efficacy of our algorithms is established via thorough experiments using commonly used node classification benchmark datasets. We also illustrate the potential practical benefit of our detection method by demonstrating its application to a real-world Bitcoin transaction network.},
 bibtype = {article},
 author = {Zhang, Yingxue and Regol, Florence and Pal, Soumyasundar and Khan, Sakif and Ma, Liheng and Coates, Mark},
 journal = {Proceedings of the 24th International Conference on Artificial Intelligence and Statistics (AISTATS)}
}

Graph neural network (GNN) models achieve superior performance when classifying nodes in graph-structured data. Given that state-of-the-art GNNs share many similarities with their CNN cousins and that CNNs suffer ad-versarial vulnerabilities, there has also been interest in exploring analogous vulnerabilities in GNNs. Indeed, recent work has demonstrated that node classification performance of several graph models, including the popular graph convolution network (GCN) model, can be severely degraded through adversar-ial perturbations to the graph structure and the node features. In this work, we take a first step towards detecting adversarial attacks against graph models. We first propose a straightforward single node threshold test for detecting nodes subject to targeted attacks. Subsequently, we describe a kernel-based two-sample test for detecting whether a given subset of nodes within a graph has been maliciously corrupted. The efficacy of our algorithms is established via thorough experiments using commonly used node classification benchmark datasets. We also illustrate the potential practical benefit of our detection method by demonstrating its application to a real-world Bitcoin transaction network.

@article{
 title = {Jointly Attacking Graph Neural Network and its Explanations},
 type = {article},
 year = {2021},
 pages = {1-17},
 websites = {http://arxiv.org/abs/2108.03388},
 id = {a8b82ff9-dafe-3e30-b1a6-570cfbc94990},
 created = {2022-01-05T09:23:16.511Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:06.090Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph Neural Networks (GNNs) have boosted the performance for many graph-related tasks. Despite the great success, recent studies have shown that GNNs are highly vulnerable to adversarial attacks, where adversaries can mislead the GNNs' prediction by modifying graphs. On the other hand, the explanation of GNNs (GNNExplainer) provides a better understanding of a trained GNN model by generating a small subgraph and features that are most influential for its prediction. In this paper, we first perform empirical studies to validate that GNNExplainer can act as an inspection tool and have the potential to detect the adversarial perturbations for graphs. This finding motivates us to further initiate a new problem investigation: Whether a graph neural network and its explanations can be jointly attacked by modifying graphs with malicious desires? It is challenging to answer this question since the goals of adversarial attacks and bypassing the GNNExplainer essentially contradict each other. In this work, we give a confirmative answer to this question by proposing a novel attack framework (GEAttack), which can attack both a GNN model and its explanations by simultaneously exploiting their vulnerabilities. Extensive experiments on two explainers (GNNExplainer and PGExplainer) under various real-world datasets demonstrate the effectiveness of the proposed method.},
 bibtype = {article},
 author = {Fan, Wenqi and Jin, Wei and Liu, Xiaorui and Xu, Han and Tang, Xianfeng and Wang, Suhang and Li, Qing and Tang, Jiliang and Wang, Jianping and Aggarwal, Charu}
}

Graph Neural Networks (GNNs) have boosted the performance for many graph-related tasks. Despite the great success, recent studies have shown that GNNs are highly vulnerable to adversarial attacks, where adversaries can mislead the GNNs' prediction by modifying graphs. On the other hand, the explanation of GNNs (GNNExplainer) provides a better understanding of a trained GNN model by generating a small subgraph and features that are most influential for its prediction. In this paper, we first perform empirical studies to validate that GNNExplainer can act as an inspection tool and have the potential to detect the adversarial perturbations for graphs. This finding motivates us to further initiate a new problem investigation: Whether a graph neural network and its explanations can be jointly attacked by modifying graphs with malicious desires? It is challenging to answer this question since the goals of adversarial attacks and bypassing the GNNExplainer essentially contradict each other. In this work, we give a confirmative answer to this question by proposing a novel attack framework (GEAttack), which can attack both a GNN model and its explanations by simultaneously exploiting their vulnerabilities. Extensive experiments on two explainers (GNNExplainer and PGExplainer) under various real-world datasets demonstrate the effectiveness of the proposed method.

@article{
 title = {Adversarial Attacks on Graph Classification via Bayesian Optimisation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.02842},
 id = {e1aebb08-4818-351c-9755-8ea1880dd92b},
 created = {2022-01-05T09:23:16.542Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:23:53.307Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Graph neural networks, a popular class of models effective in a wide range of graph-based learning tasks, have been shown to be vulnerable to adversarial attacks. While the majority of the literature focuses on such vulnerability in node-level classification tasks, little effort has been dedicated to analysing adversarial attacks on graph-level classification, an important problem with numerous real-life applications such as biochemistry and social network analysis. The few existing methods often require unrealistic setups, such as access to internal information of the victim models, or an impractically-large number of queries. We present a novel Bayesian optimisation-based attack method for graph classification models. Our method is black-box, query-efficient and parsimonious with respect to the perturbation applied. We empirically validate the effectiveness and flexibility of the proposed method on a wide range of graph classification tasks involving varying graph properties, constraints and modes of attack. Finally, we analyse common interpretable patterns behind the adversarial samples produced, which may shed further light on the adversarial robustness of graph classification models.},
 bibtype = {article},
 author = {Wan, Xingchen and Kenlay, Henry and Ru, Binxin and Blaas, Arno and Osborne, Michael A. and Dong, Xiaowen},
 number = {NeurIPS}
}

Graph neural networks, a popular class of models effective in a wide range of graph-based learning tasks, have been shown to be vulnerable to adversarial attacks. While the majority of the literature focuses on such vulnerability in node-level classification tasks, little effort has been dedicated to analysing adversarial attacks on graph-level classification, an important problem with numerous real-life applications such as biochemistry and social network analysis. The few existing methods often require unrealistic setups, such as access to internal information of the victim models, or an impractically-large number of queries. We present a novel Bayesian optimisation-based attack method for graph classification models. Our method is black-box, query-efficient and parsimonious with respect to the perturbation applied. We empirically validate the effectiveness and flexibility of the proposed method on a wide range of graph classification tasks involving varying graph properties, constraints and modes of attack. Finally, we analyse common interpretable patterns behind the adversarial samples produced, which may shed further light on the adversarial robustness of graph classification models.

@article{
 title = {Expressive 1-Lipschitz Neural Networks for Robust Multiple Graph Learning against Adversarial Attacks},
 type = {article},
 year = {2021},
 pages = {12719-12735},
 volume = {139},
 websites = {https://proceedings.mlr.press/v139/zhao21e.html},
 id = {a44cbbf2-9194-34a2-925c-c6f14589db3f},
 created = {2022-01-05T09:23:16.590Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:23:58.577Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Recent findings have shown multiple graph learning models, such as graph classification and graph matching, are highly vulnerable to adversarial attacks, i.e. small input perturbations in graph structures and node attributes can cause the model failures. Existing defense techniques often defend specific attacks on particular multiple graph learning tasks. This paper proposes an attack-agnostic graph-adaptive 1-Lipschitz neural network, ERNN, for improving the robustness of deep multiple graph learning while achieving remarkable expressive power. A K_l-Lipschitz Weibull activation function is designed to enforce the gradient norm as K_l at layer l. The nearest matrix orthogonalization and polar decomposition techniques are utilized to constraint the weight norm as 1/K_l and make the norm-constrained weight close to the original weight. The theoretical analysis is conducted to derive lower and upper bounds of feasible K_l under the 1-Lipschitz constraint. The combination of norm-constrained weight and activation function leads to the 1-Lipschitz neural network for expressive and robust multiple graph learning.},
 bibtype = {article},
 author = {Zhao, Xin and Zhang, Zeru and Zhang, Zijie and Wu, Lingfei and Jin, Jiayin and Zhou, Yang and Jin, Ruoming and Dou, Dejing and Yan, Da},
 journal = {Proceedings of the 38th International Conference on Machine Learning}
}

Recent findings have shown multiple graph learning models, such as graph classification and graph matching, are highly vulnerable to adversarial attacks, i.e. small input perturbations in graph structures and node attributes can cause the model failures. Existing defense techniques often defend specific attacks on particular multiple graph learning tasks. This paper proposes an attack-agnostic graph-adaptive 1-Lipschitz neural network, ERNN, for improving the robustness of deep multiple graph learning while achieving remarkable expressive power. A K_l-Lipschitz Weibull activation function is designed to enforce the gradient norm as K_l at layer l. The nearest matrix orthogonalization and polar decomposition techniques are utilized to constraint the weight norm as 1/K_l and make the norm-constrained weight close to the original weight. The theoretical analysis is conducted to derive lower and upper bounds of feasible K_l under the 1-Lipschitz constraint. The combination of norm-constrained weight and activation function leads to the 1-Lipschitz neural network for expressive and robust multiple graph learning.

@article{
 title = {Generating Adversarial Examples with Graph Neural Networks},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.14644},
 id = {8bc7d272-1df0-3ac9-9e64-90997b163f71},
 created = {2022-01-05T09:23:16.689Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:14.393Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Recent years have witnessed the deployment of adversarial attacks to evaluate the robustness of Neural Networks. Past work in this field has relied on traditional optimization algorithms that ignore the inherent structure of the problem and data, or generative methods that rely purely on learning and often fail to generate adversarial examples where they are hard to find. To alleviate these deficiencies, we propose a novel attack based on a graph neural network (GNN) that takes advantage of the strengths of both approaches; we call it AdvGNN. Our GNN architecture closely resembles the network we wish to attack. During inference, we perform forward-backward passes through the GNN layers to guide an iterative procedure towards adversarial examples. During training, its parameters are estimated via a loss function that encourages the efficient computation of adversarial examples over a time horizon. We show that our method beats state-of-the-art adversarial attacks, including PGD-attack, MI-FGSM, and Carlini and Wagner attack, reducing the time required to generate adversarial examples with small perturbation norms by over 65\%. Moreover, AdvGNN achieves good generalization performance on unseen networks. Finally, we provide a new challenging dataset specifically designed to allow for a more illustrative comparison of adversarial attacks.},
 bibtype = {article},
 author = {Jaeckle, Florian and Kumar, M. Pawan},
 number = {Uai}
}

Recent years have witnessed the deployment of adversarial attacks to evaluate the robustness of Neural Networks. Past work in this field has relied on traditional optimization algorithms that ignore the inherent structure of the problem and data, or generative methods that rely purely on learning and often fail to generate adversarial examples where they are hard to find. To alleviate these deficiencies, we propose a novel attack based on a graph neural network (GNN) that takes advantage of the strengths of both approaches; we call it AdvGNN. Our GNN architecture closely resembles the network we wish to attack. During inference, we perform forward-backward passes through the GNN layers to guide an iterative procedure towards adversarial examples. During training, its parameters are estimated via a loss function that encourages the efficient computation of adversarial examples over a time horizon. We show that our method beats state-of-the-art adversarial attacks, including PGD-attack, MI-FGSM, and Carlini and Wagner attack, reducing the time required to generate adversarial examples with small perturbation norms by over 65\%. Moreover, AdvGNN achieves good generalization performance on unseen networks. Finally, we provide a new challenging dataset specifically designed to allow for a more illustrative comparison of adversarial attacks.

@article{
 title = {Adversarial Attack on Large Scale Graph},
 type = {article},
 year = {2021},
 keywords = {Adversarial attack,Data models,Deep learning,Efficient attack,Graph neural networks,Measurement,Network robustness,Node classification,Perturbation methods,Robustness,Task analysis},
 pages = {1-13},
 volume = {4347},
 id = {2140e7f5-a048-316f-9ee6-e2c957881a6a},
 created = {2022-01-05T09:23:16.769Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-05T09:24:09.247Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {62c10c84-630d-4fdf-af89-e343e67460c7},
 private_publication = {false},
 abstract = {Recent studies have shown that graph neural networks (GNNs) are vulnerable against perturbations due to lack of robustness and can therefore be easily fooled. Currently, most works on attacking GNNs are mainly using gradient information to guide the attack and achieve outstanding performance. However, the high complexity of time and space makes them unmanageable for large scale graphs and becomes the major bottleneck that prevents the practical usage. We argue that the main reason is that they have to use the whole graph for attacks, resulting in the increasing time and space complexity as the data scale grows. In this work, we propose an efficient Simplified Gradient-based Attack (SGA) method to bridge this gap. SGA can cause the GNNs to misclassify specific target nodes through a multi-stage attack framework, which needs only a much smaller subgraph. In addition, we present a practical metric named Degree Assortativity Change (DAC) to measure the impacts of adversarial attacks on graph data. We evaluate our attack method on four real-world graph networks by attacking several commonly used GNNs. The experimental results demonstrate that SGA can achieve significant time and memory efficiency improvements while maintaining competitive attack performance compared to state-of-art attack techniques.},
 bibtype = {article},
 author = {Li, Jintang and Xie, Tao and Liang, Chen and Xie, Fenfang and He, Xiangnan and Zheng, Zibin},
 doi = {10.1109/TKDE.2021.3078755},
 journal = {IEEE Transactions on Knowledge and Data Engineering},
 number = {c}
}

Recent studies have shown that graph neural networks (GNNs) are vulnerable against perturbations due to lack of robustness and can therefore be easily fooled. Currently, most works on attacking GNNs are mainly using gradient information to guide the attack and achieve outstanding performance. However, the high complexity of time and space makes them unmanageable for large scale graphs and becomes the major bottleneck that prevents the practical usage. We argue that the main reason is that they have to use the whole graph for attacks, resulting in the increasing time and space complexity as the data scale grows. In this work, we propose an efficient Simplified Gradient-based Attack (SGA) method to bridge this gap. SGA can cause the GNNs to misclassify specific target nodes through a multi-stage attack framework, which needs only a much smaller subgraph. In addition, we present a practical metric named Degree Assortativity Change (DAC) to measure the impacts of adversarial attacks on graph data. We evaluate our attack method on four real-world graph networks by attacking several commonly used GNNs. The experimental results demonstrate that SGA can achieve significant time and memory efficiency improvements while maintaining competitive attack performance compared to state-of-art attack techniques.

@article{
 title = {VoxelContext-Net: An Octree based Framework for Point Cloud Compression},
 type = {article},
 year = {2021},
 pages = {6038-6047},
 id = {093ccb69-5e87-336e-95fb-91ff1f6c5ea0},
 created = {2022-01-11T11:30:16.906Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-11T11:30:23.434Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {9972d981-f25e-4229-94fb-1c4fc6296c30},
 private_publication = {false},
 abstract = {In this paper, we propose a two-stage deep learning framework called VoxelContext-Net for both static and dynamic point cloud compression. Taking advantages of both octree based methods and voxel based schemes, our approach employs the voxel context to compress the octree structured data. Specifically, we first extract the local voxel representation that encodes the spatial neighbouring context information for each node in the constructed octree. Then, in the entropy coding stage, we propose a voxel context based deep entropy model to compress the symbols of non-leaf nodes in a lossless way. Furthermore, for dynamic point cloud compression, we additionally introduce the local voxel representations from the temporal neighbouring point clouds to exploit temporal dependency. More importantly, to alleviate the distortion from the octree construction procedure, we propose a voxel context based 3D coordinate refinement method to produce more accurate reconstructed point cloud at the decoder side, which is applicable to both static and dynamic point cloud compression. The comprehensive experiments on both static and dynamic point cloud benchmark datasets(e.g., ScanNet and Semantic KITTI) clearly demonstrate the effectiveness of our newly proposed method VoxelContext-Net for 3D point cloud geometry compression.},
 bibtype = {article},
 author = {Que, Zizheng and Lu, Guo and Xu, Dong},
 doi = {10.1109/cvpr46437.2021.00598}
}

In this paper, we propose a two-stage deep learning framework called VoxelContext-Net for both static and dynamic point cloud compression. Taking advantages of both octree based methods and voxel based schemes, our approach employs the voxel context to compress the octree structured data. Specifically, we first extract the local voxel representation that encodes the spatial neighbouring context information for each node in the constructed octree. Then, in the entropy coding stage, we propose a voxel context based deep entropy model to compress the symbols of non-leaf nodes in a lossless way. Furthermore, for dynamic point cloud compression, we additionally introduce the local voxel representations from the temporal neighbouring point clouds to exploit temporal dependency. More importantly, to alleviate the distortion from the octree construction procedure, we propose a voxel context based 3D coordinate refinement method to produce more accurate reconstructed point cloud at the decoder side, which is applicable to both static and dynamic point cloud compression. The comprehensive experiments on both static and dynamic point cloud benchmark datasets(e.g., ScanNet and Semantic KITTI) clearly demonstrate the effectiveness of our newly proposed method VoxelContext-Net for 3D point cloud geometry compression.

@article{
 title = {Learning-based lossless compression of 3D point cloud geometry},
 type = {article},
 year = {2021},
 keywords = {Context model,Deep learning,G-PCC,Point cloud compression},
 pages = {4220-4224},
 volume = {2021-June},
 id = {b517e3b8-6bcf-318a-8c7f-b556bb84fd35},
 created = {2022-01-11T11:30:16.910Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-11T11:30:21.121Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {9972d981-f25e-4229-94fb-1c4fc6296c30},
 private_publication = {false},
 abstract = {This paper presents a learning-based, lossless compression method for static point cloud geometry, based on context-adaptive arithmetic coding. Unlike most existing methods working in the octree domain, our encoder operates in a hybrid mode, mixing octree and voxel-based coding. We adaptively partition the point cloud into multi-resolution voxel blocks according to the point cloud structure, and use octree to signal the partitioning. On the one hand, octree representation can eliminate the sparsity in the point cloud. On the other hand, in the voxel domain, convolutions can be naturally expressed, and geometric information (i.e., planes, surfaces, etc.) is explicitly processed by a neural network. Our context model benefits from these properties and learns a probability distribution of the voxels using a deep convolutional neural network with masked filters, called VoxelDNN. Experiments show that our method outperforms the state-of-the-art MPEG G-PCC standard with average rate savings of 28% on a diverse set of point clouds from the Microsoft Voxelized Upper Bodies (MVUB) and MPEG. The implementation is available at https://github.com/Weafre/VoxelDNN.},
 bibtype = {article},
 author = {Nguyen, Dat Thanh and Quach, Maurice and Valenzise, Giuseppe and Duhamel, Pierre},
 doi = {10.1109/ICASSP39728.2021.9414763},
 journal = {ICASSP, IEEE International Conference on Acoustics, Speech and Signal Processing - Proceedings}
}

This paper presents a learning-based, lossless compression method for static point cloud geometry, based on context-adaptive arithmetic coding. Unlike most existing methods working in the octree domain, our encoder operates in a hybrid mode, mixing octree and voxel-based coding. We adaptively partition the point cloud into multi-resolution voxel blocks according to the point cloud structure, and use octree to signal the partitioning. On the one hand, octree representation can eliminate the sparsity in the point cloud. On the other hand, in the voxel domain, convolutions can be naturally expressed, and geometric information (i.e., planes, surfaces, etc.) is explicitly processed by a neural network. Our context model benefits from these properties and learns a probability distribution of the voxels using a deep convolutional neural network with masked filters, called VoxelDNN. Experiments show that our method outperforms the state-of-the-art MPEG G-PCC standard with average rate savings of 28% on a diverse set of point clouds from the Microsoft Voxelized Upper Bodies (MVUB) and MPEG. The implementation is available at https://github.com/Weafre/VoxelDNN.

@article{
 title = {Point cloud classification with deep normalized Reeb graph convolution},
 type = {article},
 year = {2021},
 keywords = {Graph normalization,Point cloud,Reeb graph},
 pages = {104092},
 volume = {106},
 websites = {https://doi.org/10.1016/j.imavis.2020.104092},
 publisher = {Elsevier B.V.},
 id = {07fec0c5-97f6-3b4f-9413-6de62df9c445},
 created = {2022-01-14T16:04:12.051Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-05-02T08:15:04.941Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wang2021},
 private_publication = {false},
 abstract = {Recently, plenty of deep learning methods have been proposed to handle point clouds. Almost all of them input the entire point cloud and ignore the information redundancy lying in point clouds. This paper addresses this problem by extracting the Reeb graph from point clouds, which is a much more informative and compact representation of point clouds, and then filter the graph with deep graph convolution. To be able to classify or segment point clouds well, we propose (1) Graph Normalization to transform various graphs into a canonical graph space; (2) Normalized Similarity Distance to better identify the graph structure;(3) Reeb Graph Guided Node Pooling in order to aggregate high-level features from kNN graphs. Besides, our method naturally fits into the problem of classifying point clouds with unknown orientations. In the results, we show that our method gives a competitive performance to the state-of-the-art methods and outperforms previous methods by a large margin on handling point clouds with unknown orientations.},
 bibtype = {article},
 author = {Wang, Weiming and You, Yang and Liu, Wenhai and Lu, Cewu},
 doi = {10.1016/j.imavis.2020.104092},
 journal = {Image and Vision Computing}
}

Recently, plenty of deep learning methods have been proposed to handle point clouds. Almost all of them input the entire point cloud and ignore the information redundancy lying in point clouds. This paper addresses this problem by extracting the Reeb graph from point clouds, which is a much more informative and compact representation of point clouds, and then filter the graph with deep graph convolution. To be able to classify or segment point clouds well, we propose (1) Graph Normalization to transform various graphs into a canonical graph space; (2) Normalized Similarity Distance to better identify the graph structure;(3) Reeb Graph Guided Node Pooling in order to aggregate high-level features from kNN graphs. Besides, our method naturally fits into the problem of classifying point clouds with unknown orientations. In the results, we show that our method gives a competitive performance to the state-of-the-art methods and outperforms previous methods by a large margin on handling point clouds with unknown orientations.

@article{
 title = {Graph Neural Networks with Convolutional ARMA Filters},
 type = {article},
 year = {2021},
 keywords = {Chebyshev approximation,Convolution,Eigenvalues and eigenfunctions,Frequency response,Geometric Deep Learning,Graph Filters,Graph Neural Networks,Graph Signal Processing,Graph Theory,Graph neural networks,Laplace equations,Task analysis},
 pages = {1-12},
 volume = {8828},
 id = {2d88d1ab-1a7e-318f-9ed3-0bf77faa8a37},
 created = {2022-01-14T16:04:12.063Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:04:34.490Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Popular graph neural networks implement convolution operations on graphs based on polynomial spectral filters. In this paper, we propose a novel graph convolutional layer inspired by the auto-regressive moving average (ARMA) filter that, compared to polynomial ones, provides a more flexible frequency response, is more robust to noise, and better captures the global graph structure. We propose a graph neural network implementation of the ARMA filter with a recursive and distributed formulation, obtaining a convolutional layer that is efficient to train, localized in the node space, and can be transferred to new graphs at test time. We perform a spectral analysis to study the filtering effect of the proposed ARMA layer and report experiments on four downstream tasks: semi-supervised node classification, graph signal classification, graph classification, and graph regression. Results show that the proposed ARMA layer brings significant improvements over graph neural networks based on polynomial filters.},
 bibtype = {article},
 author = {Bianchi, Filippo Maria and Grattarola, Daniele and Livi, Lorenzo and Alippi, Cesare},
 doi = {10.1109/TPAMI.2021.3054830},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
 number = {c}
}

Popular graph neural networks implement convolution operations on graphs based on polynomial spectral filters. In this paper, we propose a novel graph convolutional layer inspired by the auto-regressive moving average (ARMA) filter that, compared to polynomial ones, provides a more flexible frequency response, is more robust to noise, and better captures the global graph structure. We propose a graph neural network implementation of the ARMA filter with a recursive and distributed formulation, obtaining a convolutional layer that is efficient to train, localized in the node space, and can be transferred to new graphs at test time. We perform a spectral analysis to study the filtering effect of the proposed ARMA layer and report experiments on four downstream tasks: semi-supervised node classification, graph signal classification, graph classification, and graph regression. Results show that the proposed ARMA layer brings significant improvements over graph neural networks based on polynomial filters.

@article{
 title = {On the stability of graph convolutional neural networks under edge rewiring},
 type = {article},
 year = {2021},
 keywords = {Graph convolutional neural networks,Graph signal processing,Spectral graph filters,Stability},
 pages = {8513-8517},
 volume = {2021-June},
 id = {c7e4d020-eff0-3ec3-8c05-72b666870e64},
 created = {2022-01-14T16:04:12.174Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:04:42.785Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Graph neural networks are experiencing a surge of popularity within the machine learning community due to their ability to adapt to non-Euclidean domains and instil inductive biases. Despite this, their stability, i.e., their robustness to small perturbations in the input, is not yet well understood. Although there exists some results showing the stability of graph neural networks, most take the form of an upper bound on the magnitude of change due to a perturbation in the graph topology. However, the change in the graph topology captured in existing bounds tend not to be expressed in terms of structural properties, limiting our understanding of the model robustness properties. In this work, we develop an interpretable upper bound elucidating that graph neural networks are stable to rewiring between high degree nodes. This bound and further research in bounds of similar type provide further understanding of the stability properties of graph neural networks.},
 bibtype = {article},
 author = {Kenlay, Henry and Thanou, Dorina and Dong, Xiaowen},
 doi = {10.1109/ICASSP39728.2021.9413474},
 journal = {ICASSP, IEEE International Conference on Acoustics, Speech and Signal Processing - Proceedings}
}

Graph neural networks are experiencing a surge of popularity within the machine learning community due to their ability to adapt to non-Euclidean domains and instil inductive biases. Despite this, their stability, i.e., their robustness to small perturbations in the input, is not yet well understood. Although there exists some results showing the stability of graph neural networks, most take the form of an upper bound on the magnitude of change due to a perturbation in the graph topology. However, the change in the graph topology captured in existing bounds tend not to be expressed in terms of structural properties, limiting our understanding of the model robustness properties. In this work, we develop an interpretable upper bound elucidating that graph neural networks are stable to rewiring between high degree nodes. This bound and further research in bounds of similar type provide further understanding of the stability properties of graph neural networks.

@article{
 title = {Modelling and studying the effect of graph errors in graph signal processing},
 type = {article},
 year = {2021},
 keywords = {Erdös-Rényi graphs,Error effect,Graph signal processing,Minimum distance index,Shift matrix},
 pages = {108256},
 volume = {189},
 websites = {https://doi.org/10.1016/j.sigpro.2021.108256},
 publisher = {Elsevier B.V.},
 id = {20e656de-4259-391a-ac1e-fe2251dacd0a},
 created = {2022-01-14T16:04:12.193Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:04:46.215Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {The first step for any graph signal processing (GSP) procedure is to learn the graph signal representation, i.e., to capture the dependence structure of the data into an adjacency matrix. Indeed, the adjacency matrix is typically not known a priori and has to be learned. However, it is learned with errors. A little attention has been paid to modelling such errors in the adjacency matrix, and studying their effects on GSP methods. However, modelling errors in the adjacency matrix will enable both to study the graph error effects in GSP and to develop robust GSP algorithms. In this paper, we therefore introduce practically justifiable graph error models. We also study, both analytically when possible and numerically, the graph error effect on the performance of GSP methods in different types of problems such as filtering of graph signals and independent component analysis of graph signals (graph decorrelation).},
 bibtype = {article},
 author = {Miettinen, Jari and Vorobyov, Sergiy A. and Ollila, Esa},
 doi = {10.1016/j.sigpro.2021.108256},
 journal = {Signal Processing}
}

The first step for any graph signal processing (GSP) procedure is to learn the graph signal representation, i.e., to capture the dependence structure of the data into an adjacency matrix. Indeed, the adjacency matrix is typically not known a priori and has to be learned. However, it is learned with errors. A little attention has been paid to modelling such errors in the adjacency matrix, and studying their effects on GSP methods. However, modelling errors in the adjacency matrix will enable both to study the graph error effects in GSP and to develop robust GSP algorithms. In this paper, we therefore introduce practically justifiable graph error models. We also study, both analytically when possible and numerically, the graph error effect on the performance of GSP methods in different types of problems such as filtering of graph signals and independent component analysis of graph signals (graph decorrelation).

@article{
 title = {Interpretable Stability Bounds for Spectral Graph Filters},
 type = {article},
 year = {2021},
 pages = {1-29},
 websites = {http://arxiv.org/abs/2102.09587},
 id = {ab01b562-656d-39fc-8fee-9a28720f6122},
 created = {2022-01-14T16:04:12.211Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:04:43.984Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Graph-structured data arise in a variety of real-world context ranging from sensor and transportation to biological and social networks. As a ubiquitous tool to process graph-structured data, spectral graph filters have been used to solve common tasks such as denoising and anomaly detection, as well as design deep learning architectures such as graph neural networks. Despite being an important tool, there is a lack of theoretical understanding of the stability properties of spectral graph filters, which are important for designing robust machine learning models. In this paper, we study filter stability and provide a novel and interpretable upper bound on the change of filter output, where the bound is expressed in terms of the endpoint degrees of the deleted and newly added edges, as well as the spatial proximity of those edges. This upper bound allows us to reason, in terms of structural properties of the graph, when a spectral graph filter will be stable. We further perform extensive experiments to verify intuition that can be gained from the bound.},
 bibtype = {article},
 author = {Kenlay, Henry and Thanou, Dorina and Dong, Xiaowen},
 number = {1997}
}

Graph-structured data arise in a variety of real-world context ranging from sensor and transportation to biological and social networks. As a ubiquitous tool to process graph-structured data, spectral graph filters have been used to solve common tasks such as denoising and anomaly detection, as well as design deep learning architectures such as graph neural networks. Despite being an important tool, there is a lack of theoretical understanding of the stability properties of spectral graph filters, which are important for designing robust machine learning models. In this paper, we study filter stability and provide a novel and interpretable upper bound on the change of filter output, where the bound is expressed in terms of the endpoint degrees of the deleted and newly added edges, as well as the spatial proximity of those edges. This upper bound allows us to reason, in terms of structural properties of the graph, when a spectral graph filter will be stable. We further perform extensive experiments to verify intuition that can be gained from the bound.

@article{
 title = {On the Stability of Low Pass Graph Filter With a Large Number of Edge Rewires},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2110.07234},
 id = {fa6f31ad-27dd-3580-99d4-5ccd6fab4380},
 created = {2022-01-14T16:04:12.248Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:04:47.084Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Recently, the stability of graph filters has been studied as one of the key theoretical properties driving the highly successful graph convolutional neural networks (GCNs). The stability of a graph filter characterizes the effect of topology perturbation on the output of a graph filter, a fundamental building block for GCNs. Many existing results have focused on the regime of small perturbation with a small number of edge rewires. However, the number of edge rewires can be large in many applications. To study the latter case, this work departs from the previous analysis and proves a bound on the stability of graph filter relying on the filter's frequency response. Assuming the graph filter is low pass, we show that the stability of the filter depends on perturbation to the community structure. As an application, we show that for stochastic block model graphs, the graph filter distance converges to zero when the number of nodes approaches infinity. Numerical simulations validate our findings.},
 bibtype = {article},
 author = {Nguyen, Hoang-Son and He, Yiran and Wai, Hoi-To}
}

Recently, the stability of graph filters has been studied as one of the key theoretical properties driving the highly successful graph convolutional neural networks (GCNs). The stability of a graph filter characterizes the effect of topology perturbation on the output of a graph filter, a fundamental building block for GCNs. Many existing results have focused on the regime of small perturbation with a small number of edge rewires. However, the number of edge rewires can be large in many applications. To study the latter case, this work departs from the previous analysis and proves a bound on the stability of graph filter relying on the filter's frequency response. Assuming the graph filter is low pass, we show that the stability of the filter depends on perturbation to the community structure. As an application, we show that for stochastic block model graphs, the graph filter distance converges to zero when the number of nodes approaches infinity. Numerical simulations validate our findings.

@article{
 title = {Point Cloud Processing based on Graph Neural Network},
 type = {article},
 year = {2021},
 id = {0e2e1132-4d51-3996-9ec6-85ab48424ec5},
 created = {2022-01-14T16:04:12.318Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:56:59.418Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Lin, Manxi}
}

@article{
 title = {Graph convolutional networks for graphs containing missing features},
 type = {article},
 year = {2021},
 keywords = {GCN,Graph convolutional neural network,Graph embedding,Incomplete data,Missing data,Network representation learning},
 pages = {155-168},
 volume = {117},
 websites = {https://doi.org/10.1016/j.future.2020.11.016},
 publisher = {Elsevier B.V.},
 id = {f54f1ccb-aa23-3a4c-b813-3957cbb2ddb8},
 created = {2022-01-14T16:04:12.422Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:56:59.308Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Graph Convolutional Network (GCN) has experienced great success in graph analysis tasks. It works by smoothing the node features across the graph. The current GCN models overwhelmingly assume that the node feature information is complete. However, real-world graph data are often incomplete and containing missing features. Traditionally, people have to estimate and fill in the unknown features based on imputation techniques and then apply GCN. However, the process of feature filling and graph learning are separated, resulting in degraded and unstable performance. This problem becomes more serious when a large number of features are missing. We propose an approach that adapts GCN to graphs containing missing features. In contrast to traditional strategy, our approach integrates the processing of missing features and graph learning within the same neural network architecture. Our idea is to represent the missing data by Gaussian Mixture Model (GMM) and calculate the expected activation of neurons in the first hidden layer of GCN, while keeping the other layers of the network unchanged. This enables us to learn the GMM parameters and network weight parameters in an end-to-end manner. Notably, our approach does not increase the computational complexity of GCN and it is consistent with GCN when the features are complete. We demonstrate through extensive experiments that our approach significantly outperforms the imputation based methods in node classification and link prediction tasks. We show that the performance of our approach for the case with a low level of missing features is even superior to GCN for the case with complete features.},
 bibtype = {article},
 author = {Taguchi, Hibiki and Liu, Xin and Murata, Tsuyoshi},
 doi = {10.1016/j.future.2020.11.016},
 journal = {Future Generation Computer Systems}
}

Graph Convolutional Network (GCN) has experienced great success in graph analysis tasks. It works by smoothing the node features across the graph. The current GCN models overwhelmingly assume that the node feature information is complete. However, real-world graph data are often incomplete and containing missing features. Traditionally, people have to estimate and fill in the unknown features based on imputation techniques and then apply GCN. However, the process of feature filling and graph learning are separated, resulting in degraded and unstable performance. This problem becomes more serious when a large number of features are missing. We propose an approach that adapts GCN to graphs containing missing features. In contrast to traditional strategy, our approach integrates the processing of missing features and graph learning within the same neural network architecture. Our idea is to represent the missing data by Gaussian Mixture Model (GMM) and calculate the expected activation of neurons in the first hidden layer of GCN, while keeping the other layers of the network unchanged. This enables us to learn the GMM parameters and network weight parameters in an end-to-end manner. Notably, our approach does not increase the computational complexity of GCN and it is consistent with GCN when the features are complete. We demonstrate through extensive experiments that our approach significantly outperforms the imputation based methods in node classification and link prediction tasks. We show that the performance of our approach for the case with a low level of missing features is even superior to GCN for the case with complete features.

@article{
 title = {Structural Features In Feature Space For Structure-Aware Graph Convolution},
 type = {article},
 year = {2021},
 pages = {3158-3162},
 publisher = {IEEE},
 id = {ce40d005-9b5f-367b-8dc4-adf1851ce165},
 created = {2022-01-14T16:04:12.451Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:56:59.483Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Li, Yang and Tanaka, Yuichi},
 doi = {10.1109/icip42928.2021.9506377}
}

@article{
 title = {A weakly supervised framework for real-world point cloud classification},
 type = {article},
 year = {2021},
 keywords = {point cloud classification,real-world point cloud,weakly supervised learning},
 pages = {78-88},
 volume = {102},
 websites = {https://doi.org/10.1016/j.cag.2021.12.008},
 publisher = {Elsevier Ltd.},
 id = {b869a32a-adb8-300c-a173-9f4e6b378107},
 created = {2022-01-14T16:04:12.451Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:56:59.433Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Deng, An and Wu, Yunchao and Zhang, Peng and Lu, Zhuheng and Li, Weiqing and Su, Zhiyong},
 doi = {10.1016/j.cag.2021.12.008},
 journal = {Computers & Graphics}
}

@article{
 title = {Beyond Farthest Point Sampling in Point-Wise Analysis},
 type = {article},
 year = {2021},
 pages = {1-12},
 websites = {http://arxiv.org/abs/2107.04291},
 id = {3ea300b5-8924-346c-8eb4-3cb8a2be9275},
 created = {2022-01-14T16:04:12.467Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-14T16:56:59.488Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Sampling, grouping, and aggregation are three important components in the multi-scale analysis of point clouds. In this paper, we present a novel data-driven sampler learning strategy for point-wise analysis tasks. Unlike the widely used sampling technique, Farthest Point Sampling (FPS), we propose to learn sampling and downstream applications jointly. Our key insight is that uniform sampling methods like FPS are not always optimal for different tasks: sampling more points around boundary areas can make the point-wise classification easier for segmentation. Towards the end, we propose a novel sampler learning strategy that learns sampling point displacement supervised by task-related ground truth information and can be trained jointly with the underlying tasks. We further demonstrate our methods in various point-wise analysis architectures, including semantic part segmentation, point cloud completion, and keypoint detection. Our experiments show that jointly learning of the sampler and task brings remarkable improvement over previous baseline methods.},
 bibtype = {article},
 author = {Lin, Yiqun and Chen, Lichang and Huang, Haibin and Ma, Chongyang and Han, Xiaoguang and Cui, Shuguang}
}

Sampling, grouping, and aggregation are three important components in the multi-scale analysis of point clouds. In this paper, we present a novel data-driven sampler learning strategy for point-wise analysis tasks. Unlike the widely used sampling technique, Farthest Point Sampling (FPS), we propose to learn sampling and downstream applications jointly. Our key insight is that uniform sampling methods like FPS are not always optimal for different tasks: sampling more points around boundary areas can make the point-wise classification easier for segmentation. Towards the end, we propose a novel sampler learning strategy that learns sampling point displacement supervised by task-related ground truth information and can be trained jointly with the underlying tasks. We further demonstrate our methods in various point-wise analysis architectures, including semantic part segmentation, point cloud completion, and keypoint detection. Our experiments show that jointly learning of the sampler and task brings remarkable improvement over previous baseline methods.

@article{
 title = {HISADOME, MATSUI: CASCADING FEATURE EXTRACTION FOR 3D REGISTRATION Cascading Feature Extraction for Fast Point Cloud Registration},
 type = {article},
 year = {2021},
 id = {987e36b8-78c6-3227-9fe7-a80f70b1cc46},
 created = {2022-01-18T13:44:08.877Z},
 accessed = {2022-01-18},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-18T13:44:12.563Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {a3e73416-fb6e-4382-ad74-08c4dd4c0e94},
 private_publication = {false},
 abstract = {We propose a method for speeding up a 3D point cloud registration through a cascading feature extraction. The current approach with the highest accuracy is realized by iteratively executing feature extraction and registration using deep features. However, iterative feature extraction takes time. Our proposed method significantly reduces the computational cost using cascading shallow layers. Our idea is to omit redundant computations that do not always contribute to the final accuracy. The proposed approach is approximately three times faster than the existing methods without a loss of accuracy.},
 bibtype = {article},
 author = {Hisadome, Yoichiro and Matsui, Yusuke}
}

We propose a method for speeding up a 3D point cloud registration through a cascading feature extraction. The current approach with the highest accuracy is realized by iteratively executing feature extraction and registration using deep features. However, iterative feature extraction takes time. Our proposed method significantly reduces the computational cost using cascading shallow layers. Our idea is to omit redundant computations that do not always contribute to the final accuracy. The proposed approach is approximately three times faster than the existing methods without a loss of accuracy.

@article{
 title = {A novel geometry image to accurately represent a surface by preserving mesh topology},
 type = {article},
 year = {2021},
 pages = {1-9},
 volume = {11},
 websites = {https://doi.org/10.1038/s41598-021-01722-4},
 publisher = {Nature Publishing Group UK},
 id = {388c7bf8-6c05-3d9b-9295-c851e059e2b7},
 created = {2022-01-25T12:51:06.329Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:07.563Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Zeng2021},
 folder_uuids = {10e04504-7e21-4b84-9037-5a4431df1a8a},
 private_publication = {false},
 abstract = {Geometry images parameterise a mesh with a square domain and store the information in a single chart. A one-to-one correspondence between the 2D plane and the 3D model is convenient for processing 3D models. However, the parameterised vertices are not all located at the intersection of the gridlines the existing geometry images. Thus, errors are unavoidable when a 3D mesh is reconstructed from the chart. In this paper, we propose parameterise surface onto a novel geometry image that preserves the constraint of topological neighbourhood information at integer coordinate points on a 2D grid and ensures that the shape of the reconstructed 3D mesh does not change from supplemented image data. We find a collection of edges that opens the mesh into simply connected surface with a single boundary. The point distribution with approximate blue noise spectral characteristics is computed by capacity-constrained delaunay triangulation without retriangulation. We move the vertices to the constrained mesh intersection, adjust the degenerate triangles on a regular grid, and fill the blank part by performing a local affine transformation between each triangle in the mesh and image. Unlike other geometry images, the proposed method results in no error in the reconstructed surface model when floating-point data are stored in the image. High reconstruction accuracy is achieved when the xyz positions are in a 16-bit data format in each image channel because only rounding errors exist in the topology-preserving geometry images, there are no sampling errors. This method performs one-to-one mapping between the 3D surface mesh and the points in the 2D image, while foldovers do not appear in the 2D triangular mesh, maintaining the topological structure. This also shows the potential of using a 2D image processing algorithm to process 3D models.},
 bibtype = {article},
 author = {Zeng, Sheng and Geng, Guohua and Gao, Hongjuan and Zhou, Mingquan},
 doi = {10.1038/s41598-021-01722-4},
 journal = {Scientific Reports},
 number = {1}
}

Geometry images parameterise a mesh with a square domain and store the information in a single chart. A one-to-one correspondence between the 2D plane and the 3D model is convenient for processing 3D models. However, the parameterised vertices are not all located at the intersection of the gridlines the existing geometry images. Thus, errors are unavoidable when a 3D mesh is reconstructed from the chart. In this paper, we propose parameterise surface onto a novel geometry image that preserves the constraint of topological neighbourhood information at integer coordinate points on a 2D grid and ensures that the shape of the reconstructed 3D mesh does not change from supplemented image data. We find a collection of edges that opens the mesh into simply connected surface with a single boundary. The point distribution with approximate blue noise spectral characteristics is computed by capacity-constrained delaunay triangulation without retriangulation. We move the vertices to the constrained mesh intersection, adjust the degenerate triangles on a regular grid, and fill the blank part by performing a local affine transformation between each triangle in the mesh and image. Unlike other geometry images, the proposed method results in no error in the reconstructed surface model when floating-point data are stored in the image. High reconstruction accuracy is achieved when the xyz positions are in a 16-bit data format in each image channel because only rounding errors exist in the topology-preserving geometry images, there are no sampling errors. This method performs one-to-one mapping between the 3D surface mesh and the points in the 2D image, while foldovers do not appear in the 2D triangular mesh, maintaining the topological structure. This also shows the potential of using a 2D image processing algorithm to process 3D models.

@article{
 title = {Point Cloud Classification Algorithm Based on the Fusion of the Local Binary Pattern Features and Structural Features of VoxelsLi, Y., Luo, Y., Gu, X., Chen, D., Gao, F., & Shuang, F. (2021). Point Cloud Classification Algorithm Based on the Fusion of the},
 type = {article},
 year = {2021},
 keywords = {classification,local binary pattern,point cloud,voxelization},
 pages = {3156},
 volume = {13},
 websites = {https://www.mdpi.com/2072-4292/13/16/3156/htm,https://www.mdpi.com/2072-4292/13/16/3156},
 month = {8},
 publisher = {Multidisciplinary Digital Publishing Institute},
 day = {10},
 id = {b2cabd10-7279-3df7-a58e-23f24f1962c9},
 created = {2022-01-27T08:28:39.553Z},
 accessed = {2022-01-27},
 file_attached = {false},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-27T08:36:57.492Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 private_publication = {false},
 abstract = {Point cloud classification is a key technology for point cloud applications and point cloud feature extraction is a key step towards achieving point cloud classification. Although there are many point cloud feature extraction and classification methods, and the acquisition of colored point cloud data has become easier in recent years, most point cloud processing algorithms do not consider the color information associated with the point cloud or do not make full use of the color information. Therefore, we propose a voxel-based local feature descriptor according to the voxel-based local binary pattern (VLBP) and fuses point cloud RGB information and geometric structure features using a random forest classifier to build a color point cloud classification algorithm. The proposed algorithm voxelizes the point cloud; divides the neighborhood of the center point into cubes (i.e., multiple adjacent sub-voxels); compares the gray information of the voxel center and adjacent sub-voxels; performs voxel global thresholding to convert it into a binary code; and uses a local difference sign–magnitude transform (LDSMT) to decompose the local difference of an entire voxel into two complementary components of sign and magnitude. Then, the VLBP feature of each point is extracted. To obtain more structural information about the point cloud, the proposed method extracts the normal vector of each point and the corresponding fast point feature histogram (FPFH) based on the normal vector. Finally, the geometric mechanism features (normal vector and FPFH) and color features (RGB and VLBP features) of the point cloud are fused, and a random forest classifier is used to classify the color laser point cloud. The experimental results show that the proposed algorithm can achieve effective point cloud classification for point cloud data from different indoor and outdoor scenes, and the proposed VLBP features can improve the accuracy of point cloud classification.},
 bibtype = {article},
 author = {Li, Yong and Luo, Yinzheng and Gu, Xia and Chen, Dong and Gao, Fang and Shuang, Feng},
 doi = {10.3390/RS13163156},
 journal = {Remote Sensing 2021, Vol. 13, Page 3156},
 number = {16}
}

Point cloud classification is a key technology for point cloud applications and point cloud feature extraction is a key step towards achieving point cloud classification. Although there are many point cloud feature extraction and classification methods, and the acquisition of colored point cloud data has become easier in recent years, most point cloud processing algorithms do not consider the color information associated with the point cloud or do not make full use of the color information. Therefore, we propose a voxel-based local feature descriptor according to the voxel-based local binary pattern (VLBP) and fuses point cloud RGB information and geometric structure features using a random forest classifier to build a color point cloud classification algorithm. The proposed algorithm voxelizes the point cloud; divides the neighborhood of the center point into cubes (i.e., multiple adjacent sub-voxels); compares the gray information of the voxel center and adjacent sub-voxels; performs voxel global thresholding to convert it into a binary code; and uses a local difference sign–magnitude transform (LDSMT) to decompose the local difference of an entire voxel into two complementary components of sign and magnitude. Then, the VLBP feature of each point is extracted. To obtain more structural information about the point cloud, the proposed method extracts the normal vector of each point and the corresponding fast point feature histogram (FPFH) based on the normal vector. Finally, the geometric mechanism features (normal vector and FPFH) and color features (RGB and VLBP features) of the point cloud are fused, and a random forest classifier is used to classify the color laser point cloud. The experimental results show that the proposed algorithm can achieve effective point cloud classification for point cloud data from different indoor and outdoor scenes, and the proposed VLBP features can improve the accuracy of point cloud classification.

@article{
 title = {Point Cloud Classification Algorithm Based on the Fusion of the Local Binary Pattern Features and Structural Features of Voxels},
 type = {article},
 year = {2021},
 keywords = {classification,local binary pattern,point cloud,voxelization},
 pages = {3156},
 volume = {13},
 websites = {https://www.mdpi.com/2072-4292/13/16/3156/htm,https://www.mdpi.com/2072-4292/13/16/3156},
 month = {8},
 publisher = {Multidisciplinary Digital Publishing Institute},
 day = {10},
 id = {38167400-5c02-3ba6-af8f-22ec4d841776},
 created = {2022-01-27T08:36:17.922Z},
 accessed = {2022-01-27},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-01-28T07:27:10.300Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {a3e73416-fb6e-4382-ad74-08c4dd4c0e94},
 private_publication = {false},
 abstract = {Point cloud classification is a key technology for point cloud applications and point cloud feature extraction is a key step towards achieving point cloud classification. Although there are many point cloud feature extraction and classification methods, and the acquisition of colored point cloud data has become easier in recent years, most point cloud processing algorithms do not consider the color information associated with the point cloud or do not make full use of the color information. Therefore, we propose a voxel-based local feature descriptor according to the voxel-based local binary pattern (VLBP) and fuses point cloud RGB information and geometric structure features using a random forest classifier to build a color point cloud classification algorithm. The proposed algorithm voxelizes the point cloud; divides the neighborhood of the center point into cubes (i.e., multiple adjacent sub-voxels); compares the gray information of the voxel center and adjacent sub-voxels; performs voxel global thresholding to convert it into a binary code; and uses a local difference sign–magnitude transform (LDSMT) to decompose the local difference of an entire voxel into two complementary components of sign and magnitude. Then, the VLBP feature of each point is extracted. To obtain more structural information about the point cloud, the proposed method extracts the normal vector of each point and the corresponding fast point feature histogram (FPFH) based on the normal vector. Finally, the geometric mechanism features (normal vector and FPFH) and color features (RGB and VLBP features) of the point cloud are fused, and a random forest classifier is used to classify the color laser point cloud. The experimental results show that the proposed algorithm can achieve effective point cloud classification for point cloud data from different indoor and outdoor scenes, and the proposed VLBP features can improve the accuracy of point cloud classification.},
 bibtype = {article},
 author = {Li, Yong and Luo, Yinzheng and Gu, Xia and Chen, Dong and Gao, Fang and Shuang, Feng},
 doi = {10.3390/RS13163156},
 journal = {Remote Sensing 2021, Vol. 13, Page 3156},
 number = {16}
}

Point cloud classification is a key technology for point cloud applications and point cloud feature extraction is a key step towards achieving point cloud classification. Although there are many point cloud feature extraction and classification methods, and the acquisition of colored point cloud data has become easier in recent years, most point cloud processing algorithms do not consider the color information associated with the point cloud or do not make full use of the color information. Therefore, we propose a voxel-based local feature descriptor according to the voxel-based local binary pattern (VLBP) and fuses point cloud RGB information and geometric structure features using a random forest classifier to build a color point cloud classification algorithm. The proposed algorithm voxelizes the point cloud; divides the neighborhood of the center point into cubes (i.e., multiple adjacent sub-voxels); compares the gray information of the voxel center and adjacent sub-voxels; performs voxel global thresholding to convert it into a binary code; and uses a local difference sign–magnitude transform (LDSMT) to decompose the local difference of an entire voxel into two complementary components of sign and magnitude. Then, the VLBP feature of each point is extracted. To obtain more structural information about the point cloud, the proposed method extracts the normal vector of each point and the corresponding fast point feature histogram (FPFH) based on the normal vector. Finally, the geometric mechanism features (normal vector and FPFH) and color features (RGB and VLBP features) of the point cloud are fused, and a random forest classifier is used to classify the color laser point cloud. The experimental results show that the proposed algorithm can achieve effective point cloud classification for point cloud data from different indoor and outdoor scenes, and the proposed VLBP features can improve the accuracy of point cloud classification.

@article{
 title = {CoFiNet: Reliable Coarse-to-fine Correspondences for Robust Point Cloud Registration},
 type = {article},
 year = {2021},
 pages = {1-18},
 websites = {http://arxiv.org/abs/2110.14076},
 id = {863ecfba-ba3e-3035-bdd4-d0a6e4a41813},
 created = {2022-02-03T11:00:45.137Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-01-10T11:25:26.167Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d54ba66b-a8cf-41de-8e2d-c3256f322e07},
 private_publication = {false},
 abstract = {We study the problem of extracting correspondences between a pair of point clouds for registration. For correspondence retrieval, existing works benefit from matching sparse keypoints detected from dense points but usually struggle to guarantee their repeatability. To address this issue, we present CoFiNet - Coarse-to-Fine Network which extracts hierarchical correspondences from coarse to fine without keypoint detection. On a coarse scale and guided by a weighting scheme, our model firstly learns to match down-sampled nodes whose vicinity points share more overlap, which significantly shrinks the search space of a consecutive stage. On a finer scale, node proposals are consecutively expanded to patches that consist of groups of points together with associated descriptors. Point correspondences are then refined from the overlap areas of corresponding patches, by a density-adaptive matching module capable to deal with varying point density. Extensive evaluation of CoFiNet on both indoor and outdoor standard benchmarks shows our superiority over existing methods. Especially on 3DLoMatch where point clouds share less overlap, CoFiNet significantly outperforms state-of-the-art approaches by at least 5% on Registration Recall, with at most two-third of their parameters.},
 bibtype = {article},
 author = {Yu, Hao and Li, Fu and Saleh, Mahdi and Busam, Benjamin and Ilic, Slobodan},
 number = {NeurIPS}
}

We study the problem of extracting correspondences between a pair of point clouds for registration. For correspondence retrieval, existing works benefit from matching sparse keypoints detected from dense points but usually struggle to guarantee their repeatability. To address this issue, we present CoFiNet - Coarse-to-Fine Network which extracts hierarchical correspondences from coarse to fine without keypoint detection. On a coarse scale and guided by a weighting scheme, our model firstly learns to match down-sampled nodes whose vicinity points share more overlap, which significantly shrinks the search space of a consecutive stage. On a finer scale, node proposals are consecutively expanded to patches that consist of groups of points together with associated descriptors. Point correspondences are then refined from the overlap areas of corresponding patches, by a density-adaptive matching module capable to deal with varying point density. Extensive evaluation of CoFiNet on both indoor and outdoor standard benchmarks shows our superiority over existing methods. Especially on 3DLoMatch where point clouds share less overlap, CoFiNet significantly outperforms state-of-the-art approaches by at least 5% on Registration Recall, with at most two-third of their parameters.

@article{
 title = {PREDATOR: Registration of 3D Point Clouds with Low Overlap},
 type = {article},
 year = {2021},
 pages = {4265-4274},
 id = {eea6d683-a934-3d66-984e-7f7abbdba69a},
 created = {2022-02-03T11:00:45.143Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-03T11:00:56.149Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d54ba66b-a8cf-41de-8e2d-c3256f322e07},
 private_publication = {false},
 abstract = {We introduce PREDATOR, a model for pairwise point-cloud registration with deep attention to the overlap region. Different from previous work, our model is specifically designed to handle (also) point-cloud pairs with low overlap. Its key novelty is an overlap-attention block for early information exchange between the latent encodings of the two point clouds. In this way the subsequent decoding of the latent representations into per-point features is conditioned on the respective other point cloud, and thus can predict which points are not only salient, but also lie in the overlap region between the two point clouds. The ability to focus on points that are relevant for matching greatly improves performance: PREDATOR raises the rate of successful registrations by more than 20% in the low-overlap scenario, and also sets a new state of the art for the 3DMatch benchmark with 89% registration recall. [Code release].},
 bibtype = {article},
 author = {Huang, Shengyu and Gojcic, Zan and Usvyatsov, Mikhail and Wieser, Andreas and Schindler, Konrad},
 doi = {10.1109/CVPR46437.2021.00425},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

We introduce PREDATOR, a model for pairwise point-cloud registration with deep attention to the overlap region. Different from previous work, our model is specifically designed to handle (also) point-cloud pairs with low overlap. Its key novelty is an overlap-attention block for early information exchange between the latent encodings of the two point clouds. In this way the subsequent decoding of the latent representations into per-point features is conditioned on the respective other point cloud, and thus can predict which points are not only salient, but also lie in the overlap region between the two point clouds. The ability to focus on points that are relevant for matching greatly improves performance: PREDATOR raises the rate of successful registrations by more than 20% in the low-overlap scenario, and also sets a new state of the art for the 3DMatch benchmark with 89% registration recall. [Code release].

@article{
 title = {Learning of 3D Graph Convolution Networks for Point Cloud Analysis},
 type = {article},
 year = {2021},
 keywords = {3D Classification,3D Segmentation,3D vision,Convolution,Deformable Kernels,Feature extraction,Graph Convolution Networks,Kernel,Point Clouds,Shape,Task analysis,Three-dimensional displays,Two dimensional displays},
 volume = {X},
 id = {022a2dac-6b5b-337e-aa77-aff6b65b7b4e},
 created = {2022-02-15T11:01:25.914Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-15T11:03:54.102Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8315fdc0-e3a9-47f0-9186-21b3433d86d2},
 private_publication = {false},
 abstract = {Point clouds are among the popular geometry representations in 3D vision. However, unlike 2D images with pixel-wise layouts, such representations containing unordered data points which make the processing and understanding the associated semantic information quite challenging. Although a number of previous works attempt to analyze point clouds and achieve promising performances, their performances would degrade significantly when data variations like shift and scale changes are presented. In this paper, we propose 3D Graph Convolution Networks (3D-GCN), which uniquely learns 3D kernels with graph max-pooling mechanisms for extracting geometric features from point cloud data across different scales. We show that, with the proposed 3D-GCN, satisfactory shift and scale invariance can be jointly achieved. We show that 3D-GCN can be applied to point cloud classification and segmentation tasks, with ablation studies and visualizations verifying the design of 3D-GCN.},
 bibtype = {article},
 author = {Lin, Zhi Hao and Huang, Sheng Yu and Wang, Yu Chiang Frank},
 doi = {10.1109/TPAMI.2021.3059758},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
 number = {X}
}

Point clouds are among the popular geometry representations in 3D vision. However, unlike 2D images with pixel-wise layouts, such representations containing unordered data points which make the processing and understanding the associated semantic information quite challenging. Although a number of previous works attempt to analyze point clouds and achieve promising performances, their performances would degrade significantly when data variations like shift and scale changes are presented. In this paper, we propose 3D Graph Convolution Networks (3D-GCN), which uniquely learns 3D kernels with graph max-pooling mechanisms for extracting geometric features from point cloud data across different scales. We show that, with the proposed 3D-GCN, satisfactory shift and scale invariance can be jointly achieved. We show that 3D-GCN can be applied to point cloud classification and segmentation tasks, with ablation studies and visualizations verifying the design of 3D-GCN.

@article{
 title = {Sensor agnostic semantic segmentation of structurally diverse and complex forest point clouds using deep learning},
 type = {article},
 year = {2021},
 keywords = {Automated,Deep learning,Digital terrain model,Forest,LiDAR,Photogrammetry,Point cloud,Segmentation,Structure from motion,Terrestrial laser scanning},
 volume = {13},
 id = {9af8f017-b62d-3154-b8b8-e6f579fbc4ef},
 created = {2022-02-21T06:44:13.997Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-22T12:22:24.332Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e523c16b-0594-4b52-9c4e-9052fcb9dbed},
 private_publication = {false},
 abstract = {Forest inventories play an important role in enabling informed decisions to be made for the management and conservation of forest resources; however, the process of collecting inventory information is laborious. Despite advancements in mapping technologies allowing forests to be digitized in finer granularity than ever before, it is still common for forest measurements to be collected using simple tools such as calipers, measuring tapes, and hypsometers. Dense understory vegetation and complex forest structures can present substantial challenges to point cloud processing tools, often leading to erroneous measurements, and making them of less utility in complex forests. To address this challenge, this research demonstrates an effective deep learning approach for seman-tically segmenting high-resolution forest point clouds from multiple different sensing systems in diverse forest conditions. Seven diverse point cloud datasets were manually segmented to train and evaluate this model, resulting in per-class segmentation accuracies of Terrain: 95.92%, Vegetation: 96.02%, Coarse Woody Debris: 54.98%, and Stem: 96.09%. By exploiting the segmented point cloud, we also present a method of extracting a Digital Terrain Model (DTM) from such segmented point clouds. This approach was applied to a set of six point clouds that were made publicly available as part of a benchmarking study to evaluate the DTM performance. The mean DTM error was 0.04 m relative to the reference with 99.9% completeness. These approaches serve as useful steps toward a fully automated and reliable measurement extraction tool, agnostic to the sensing technology used or the complexity of the forest, provided that the point cloud has sufficient coverage and accuracy. Ongoing work will see these models incorporated into a fully automated forest measurement tool for the extraction of structural metrics for applications in forestry, conservation, and research.},
 bibtype = {article},
 author = {Krisanski, Sean and Taskhiri, Mohammad Sadegh and Aracil, Susana Gonzalez and Herries, David and Turner, Paul},
 doi = {10.3390/rs13081413},
 journal = {Remote Sensing},
 number = {8}
}

Forest inventories play an important role in enabling informed decisions to be made for the management and conservation of forest resources; however, the process of collecting inventory information is laborious. Despite advancements in mapping technologies allowing forests to be digitized in finer granularity than ever before, it is still common for forest measurements to be collected using simple tools such as calipers, measuring tapes, and hypsometers. Dense understory vegetation and complex forest structures can present substantial challenges to point cloud processing tools, often leading to erroneous measurements, and making them of less utility in complex forests. To address this challenge, this research demonstrates an effective deep learning approach for seman-tically segmenting high-resolution forest point clouds from multiple different sensing systems in diverse forest conditions. Seven diverse point cloud datasets were manually segmented to train and evaluate this model, resulting in per-class segmentation accuracies of Terrain: 95.92%, Vegetation: 96.02%, Coarse Woody Debris: 54.98%, and Stem: 96.09%. By exploiting the segmented point cloud, we also present a method of extracting a Digital Terrain Model (DTM) from such segmented point clouds. This approach was applied to a set of six point clouds that were made publicly available as part of a benchmarking study to evaluate the DTM performance. The mean DTM error was 0.04 m relative to the reference with 99.9% completeness. These approaches serve as useful steps toward a fully automated and reliable measurement extraction tool, agnostic to the sensing technology used or the complexity of the forest, provided that the point cloud has sufficient coverage and accuracy. Ongoing work will see these models incorporated into a fully automated forest measurement tool for the extraction of structural metrics for applications in forestry, conservation, and research.

@article{
 title = {Point Set Voting for Partial Point Cloud Analysis},
 type = {article},
 year = {2021},
 keywords = {Deep learning methods},
 pages = {596-603},
 volume = {6},
 id = {6659abf4-1156-3071-89a0-505868e87084},
 created = {2022-02-21T06:44:14.251Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-22T12:22:24.443Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e523c16b-0594-4b52-9c4e-9052fcb9dbed},
 private_publication = {false},
 abstract = {The continual improvement of 3D sensors has driven the development of algorithms to perform point cloud analysis. In fact, techniques for point cloud classification and segmentation have in recent years achieved incredible performance driven in part by leveraging large synthetic datasets. Unfortunately these same state-of-The-Art approaches perform poorly when applied to incomplete point clouds. This limitation of existing algorithms is particularly concerning since point clouds generated by 3D sensors in the real world are usually incomplete due to perspective view or occlusion by other objects. This paper proposes a general model for partial point clouds analysis wherein the latent feature encoding a complete point cloud is inferred by applying a point set voting strategy. In particular, each local point set constructs a vote that corresponds to a distribution in the latent space, and the optimal latent feature is the one with the highest probability. This approach ensures that any subsequent point cloud analysis is robust to partial observation while simultaneously guaranteeing that the proposed model is able to output multiple possible results. This paper illustrates that this proposed method achieves the state-of-The-Art performance on shape classification, part segmentation and point cloud completion.},
 bibtype = {article},
 author = {Zhang, Junming and Chen, Weijia and Wang, Yuping and Vasudevan, Ram and Johnson-Roberson, Matthew},
 doi = {10.1109/LRA.2020.3048658},
 journal = {IEEE Robotics and Automation Letters},
 number = {2}
}

The continual improvement of 3D sensors has driven the development of algorithms to perform point cloud analysis. In fact, techniques for point cloud classification and segmentation have in recent years achieved incredible performance driven in part by leveraging large synthetic datasets. Unfortunately these same state-of-The-Art approaches perform poorly when applied to incomplete point clouds. This limitation of existing algorithms is particularly concerning since point clouds generated by 3D sensors in the real world are usually incomplete due to perspective view or occlusion by other objects. This paper proposes a general model for partial point clouds analysis wherein the latent feature encoding a complete point cloud is inferred by applying a point set voting strategy. In particular, each local point set constructs a vote that corresponds to a distribution in the latent space, and the optimal latent feature is the one with the highest probability. This approach ensures that any subsequent point cloud analysis is robust to partial observation while simultaneously guaranteeing that the proposed model is able to output multiple possible results. This paper illustrates that this proposed method achieves the state-of-The-Art performance on shape classification, part segmentation and point cloud completion.

@article{
 title = {Wireless 3D Point Cloud Delivery Using Deep Graph Neural Networks},
 type = {article},
 year = {2021},
 keywords = {Point cloud,deep graph neural network},
 pages = {1-6},
 id = {45a05c6a-37e0-38c6-ba8b-0cf8147cd384},
 created = {2022-02-21T06:44:14.389Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T07:18:43.492Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449,e523c16b-0594-4b52-9c4e-9052fcb9dbed,a6e140dd-a959-4148-9ed8-b29d0c7966c6},
 private_publication = {false},
 abstract = {In typical point cloud delivery, a sender uses octree-based and graph-based digital video compression to send three-dimensional (3D) points and color attributes. However, the digital-based schemes have an issue called the cliff effect, where the 3D reconstruction quality will be a step function in terms of wireless channel quality. To prevent the cliff effect subject to channel quality fluctuation, we have proposed a wireless point cloud delivery called HoloCast inspired by soft delivery. Although the HoloCast realizes graceful quality improvement according to instantaneous wireless channel quality, it requires large communication overheads. In this paper, we propose a novel scheme for soft point cloud delivery to simultaneously realize better 3D reconstruction quality and lower communication overheads. The proposed scheme introduces an end-to-end deep learning framework based on graph neural network (GNN) to reconstruct high-quality point clouds from its distorted observation under wireless fading channels. We demonstrate that the proposed GNN-based scheme can reconstruct a clean 3D point cloud with low overheads by removing fading and noise effects.},
 bibtype = {article},
 author = {Fujihashi, Takuya and Koike-Akino, Toshiaki and Chen, Siheng and Watanabe, Takashi},
 doi = {10.1109/ICC42927.2021.9500925},
 journal = {IEEE International Conference on Communications}
}

In typical point cloud delivery, a sender uses octree-based and graph-based digital video compression to send three-dimensional (3D) points and color attributes. However, the digital-based schemes have an issue called the cliff effect, where the 3D reconstruction quality will be a step function in terms of wireless channel quality. To prevent the cliff effect subject to channel quality fluctuation, we have proposed a wireless point cloud delivery called HoloCast inspired by soft delivery. Although the HoloCast realizes graceful quality improvement according to instantaneous wireless channel quality, it requires large communication overheads. In this paper, we propose a novel scheme for soft point cloud delivery to simultaneously realize better 3D reconstruction quality and lower communication overheads. The proposed scheme introduces an end-to-end deep learning framework based on graph neural network (GNN) to reconstruct high-quality point clouds from its distorted observation under wireless fading channels. We demonstrate that the proposed GNN-based scheme can reconstruct a clean 3D point cloud with low overheads by removing fading and noise effects.

@article{
 title = {Point2color: 3D point cloud colorization using a conditional generative network and differentiable rendering for airborne LiDAR},
 type = {article},
 year = {2021},
 pages = {1062-1071},
 id = {fc6ec6e7-ec07-32aa-a371-8a00eef51ce0},
 created = {2022-02-21T06:44:14.573Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-22T09:08:42.725Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e523c16b-0594-4b52-9c4e-9052fcb9dbed},
 private_publication = {false},
 abstract = {Airborne LiDAR observations are very effective for providing accurate 3D point clouds, and archived data are becoming available to the public. In many cases, only geometric information is available in the published 3D point cloud observed by airborne LiDAR (airborne 3D point cloud), and geometric information alone is not readable. Thus, it is important to colorize airborne 3D point clouds to improve visual readability. A scheme for 3D point cloud colorization using a conditional generative adversarial network (cGAN) was proposed, but it is difficult to apply to airborne LiDAR because the method is for artificial CAD models. Since airborne 3D point clouds are spread over a wider area than simple CAD models, it is important to evaluate them spatially in two-dimensional (2D) images. Currently, the differentiable renderer is the most reliable method to bridge 3D and 2D images. In this paper, we propose an airborne 3D point cloud colorization scheme called point2color using cGAN with points and rendered images. To achieve airborne 3D point cloud colorization, we estimate the color of each point with PointNet++ and render the estimated colored airborne 3D point cloud into a 2D image with a differentiable renderer. The network is then trained by minimizing the distance between real color and colorized fake color. The experimental results demonstrate the effectiveness of point2color using the IEEE GRSS 2018 Data Fusion Contest dataset with lower error than previous studies. Furthermore, an ablation study demonstrates the effectiveness of using a cGAN pipeline and 2D images via a differentiable renderer. Our code will be available at GitHub.},
 bibtype = {article},
 author = {Shinohara, Takayuki and Xiu, Haoyi and Matsuoka, Masashi},
 doi = {10.1109/CVPRW53098.2021.00117},
 journal = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops}
}

Airborne LiDAR observations are very effective for providing accurate 3D point clouds, and archived data are becoming available to the public. In many cases, only geometric information is available in the published 3D point cloud observed by airborne LiDAR (airborne 3D point cloud), and geometric information alone is not readable. Thus, it is important to colorize airborne 3D point clouds to improve visual readability. A scheme for 3D point cloud colorization using a conditional generative adversarial network (cGAN) was proposed, but it is difficult to apply to airborne LiDAR because the method is for artificial CAD models. Since airborne 3D point clouds are spread over a wider area than simple CAD models, it is important to evaluate them spatially in two-dimensional (2D) images. Currently, the differentiable renderer is the most reliable method to bridge 3D and 2D images. In this paper, we propose an airborne 3D point cloud colorization scheme called point2color using cGAN with points and rendered images. To achieve airborne 3D point cloud colorization, we estimate the color of each point with PointNet++ and render the estimated colored airborne 3D point cloud into a 2D image with a differentiable renderer. The network is then trained by minimizing the distance between real color and colorized fake color. The experimental results demonstrate the effectiveness of point2color using the IEEE GRSS 2018 Data Fusion Contest dataset with lower error than previous studies. Furthermore, an ablation study demonstrates the effectiveness of using a cGAN pipeline and 2D images via a differentiable renderer. Our code will be available at GitHub.

@article{
 title = {Wireless 3D Point Cloud Delivery Using Deep Graph Neural Networks},
 type = {article},
 year = {2021},
 keywords = {Point cloud,deep graph neural network},
 pages = {0-5},
 publisher = {IEEE},
 id = {ff072634-7be1-350f-8f1f-a1b7c6f1eb5a},
 created = {2022-02-23T06:26:16.728Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T07:18:43.225Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449,e523c16b-0594-4b52-9c4e-9052fcb9dbed,a6e140dd-a959-4148-9ed8-b29d0c7966c6},
 private_publication = {false},
 abstract = {In typical point cloud delivery, a sender uses octree-based and graph-based digital video compression to send three-dimensional (3D) points and color attributes. However, the digital-based schemes have an issue called the cliff effect, where the 3D reconstruction quality will be a step function in terms of wireless channel quality. To prevent the cliff effect subject to channel quality fluctuation, we have proposed a wireless point cloud delivery called HoloCast inspired by soft delivery. Although the HoloCast realizes graceful quality improvement according to instantaneous wireless channel quality, it requires large communication overheads. In this paper, we propose a novel scheme for soft point cloud delivery to simultaneously realize better 3D reconstruction quality and lower communication overheads. The proposed scheme introduces an end-to-end deep learning framework based on graph neural network (GNN) to reconstruct high-quality point clouds from its distorted observation under wireless fading channels. We demonstrate that the proposed GNN-based scheme can reconstruct a clean 3D point cloud with low overheads by removing fading and noise effects.},
 bibtype = {article},
 author = {Fujihashi, Takuya and Koike-Akino, Toshiaki and Chen, Siheng and Watanabe, Takashi},
 doi = {10.1109/ICC42927.2021.9500925},
 journal = {IEEE International Conference on Communications}
}

In typical point cloud delivery, a sender uses octree-based and graph-based digital video compression to send three-dimensional (3D) points and color attributes. However, the digital-based schemes have an issue called the cliff effect, where the 3D reconstruction quality will be a step function in terms of wireless channel quality. To prevent the cliff effect subject to channel quality fluctuation, we have proposed a wireless point cloud delivery called HoloCast inspired by soft delivery. Although the HoloCast realizes graceful quality improvement according to instantaneous wireless channel quality, it requires large communication overheads. In this paper, we propose a novel scheme for soft point cloud delivery to simultaneously realize better 3D reconstruction quality and lower communication overheads. The proposed scheme introduces an end-to-end deep learning framework based on graph neural network (GNN) to reconstruct high-quality point clouds from its distorted observation under wireless fading channels. We demonstrate that the proposed GNN-based scheme can reconstruct a clean 3D point cloud with low overheads by removing fading and noise effects.

@article{
 title = {BuildingNet: Learning to Label 3D Buildings},
 type = {article},
 year = {2021},
 pages = {10397-10407},
 websites = {http://arxiv.org/abs/2110.04955},
 id = {fb3721b1-5df7-34e0-b70c-a5ce05679cc8},
 created = {2022-02-24T07:44:06.302Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-02-24T07:44:16.109Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5},
 private_publication = {false},
 abstract = {We introduce BuildingNet: (a) a large-scale dataset of 3D building models whose exteriors are consistently labeled, (b) a graph neural network that labels building meshes by analyzing spatial and structural relations of their geometric primitives. To create our dataset, we used crowdsourcing combined with expert guidance, resulting in 513K annotated mesh primitives, grouped into 292K semantic part components across 2K building models. The dataset covers several building categories, such as houses, churches, skyscrapers, town halls, libraries, and castles. We include a benchmark for evaluating mesh and point cloud labeling. Buildings have more challenging structural complexity compared to objects in existing benchmarks (e.g., ShapeNet, PartNet), thus, we hope that our dataset can nurture the development of algorithms that are able to cope with such large-scale geometric data for both vision and graphics tasks e.g., 3D semantic segmentation, part-based generative models, correspondences, texturing, and analysis of point cloud data acquired from real-world buildings. Finally, we show that our mesh-based graph neural network significantly improves performance over several baselines for labeling 3D meshes.},
 bibtype = {article},
 author = {Selvaraju, Pratheba and Nabail, Mohamed and Loizou, Marios and Maslioukova, Maria and Averkiou, Melinos and Andreou, Andreas and Chaudhuri, Siddhartha and Kalogerakis, Evangelos}
}

We introduce BuildingNet: (a) a large-scale dataset of 3D building models whose exteriors are consistently labeled, (b) a graph neural network that labels building meshes by analyzing spatial and structural relations of their geometric primitives. To create our dataset, we used crowdsourcing combined with expert guidance, resulting in 513K annotated mesh primitives, grouped into 292K semantic part components across 2K building models. The dataset covers several building categories, such as houses, churches, skyscrapers, town halls, libraries, and castles. We include a benchmark for evaluating mesh and point cloud labeling. Buildings have more challenging structural complexity compared to objects in existing benchmarks (e.g., ShapeNet, PartNet), thus, we hope that our dataset can nurture the development of algorithms that are able to cope with such large-scale geometric data for both vision and graphics tasks e.g., 3D semantic segmentation, part-based generative models, correspondences, texturing, and analysis of point cloud data acquired from real-world buildings. Finally, we show that our mesh-based graph neural network significantly improves performance over several baselines for labeling 3D meshes.

@article{
 title = {An Overview on the Application of Graph Neural Networks in Wireless Networks},
 type = {article},
 year = {2021},
 keywords = {Wireless networks,graph neural networks,resource management},
 pages = {2547-2565},
 volume = {2},
 publisher = {IEEE},
 id = {2bfff17f-4ce7-34d5-bbd6-13bdfdef956a},
 created = {2022-03-01T12:39:38.016Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-02T07:52:11.588Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {In recent years, with the rapid enhancement of computing power, deep learning methods have been widely applied in wireless networks and achieved impressive performance. To effectively exploit the information of graph-structured data as well as contextual information, graph neural networks (GNNs) have been introduced to address a series of optimization problems of wireless networks. In this overview, we first illustrate the construction method of wireless communication graph for various wireless networks and simply introduce the progress of several classical paradigms of GNNs. Then, several applications of GNNs in wireless networks such as resource allocation and several emerging fields, are discussed in detail. Finally, some research trends about the applications of GNNs in wireless communication systems are discussed.},
 bibtype = {article},
 author = {He, Shiwen and Xiong, Shaowen and Ou, Yeyu and Zhang, Jian and Wang, Jiaheng and Huang, Yongming and Zhang, Yaoxue},
 doi = {10.1109/OJCOMS.2021.3128637},
 journal = {IEEE Open Journal of the Communications Society},
 number = {November}
}

In recent years, with the rapid enhancement of computing power, deep learning methods have been widely applied in wireless networks and achieved impressive performance. To effectively exploit the information of graph-structured data as well as contextual information, graph neural networks (GNNs) have been introduced to address a series of optimization problems of wireless networks. In this overview, we first illustrate the construction method of wireless communication graph for various wireless networks and simply introduce the progress of several classical paradigms of GNNs. Then, several applications of GNNs in wireless networks such as resource allocation and several emerging fields, are discussed in detail. Finally, some research trends about the applications of GNNs in wireless communication systems are discussed.

@article{
 title = {Graph Neural Networks with Parallel Neighborhood Aggregations for Graph Classification},
 type = {article},
 year = {2021},
 pages = {1-20},
 websites = {http://arxiv.org/abs/2111.11482},
 id = {89dcaa2c-c6dd-3c34-944d-aa35414abfaa},
 created = {2022-03-01T12:39:38.070Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-01T12:39:42.792Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {We focus on graph classification using a graph neural network (GNN) model that precomputes the node features using a bank of neighborhood aggregation graph operators arranged in parallel. These GNN models have a natural advantage of reduced training and inference time due to the precomputations but are also fundamentally different from popular GNN variants that update node features through a sequential neighborhood aggregation procedure during training. We provide theoretical conditions under which a generic GNN model with parallel neighborhood aggregations (PA-GNNs, in short) are provably as powerful as the well-known Weisfeiler-Lehman (WL) graph isomorphism test in discriminating non-isomorphic graphs. Although PA-GNN models do not have an apparent relationship with the WL test, we show that the graph embeddings obtained from these two methods are injectively related. We then propose a specialized PA-GNN model, called SPIN, which obeys the developed conditions. We demonstrate via numerical experiments that the developed model achieves state-of-the-art performance on many diverse real-world datasets while maintaining the discriminative power of the WL test and the computational advantage of preprocessing graphs before the training process.},
 bibtype = {article},
 author = {Doshi, Siddhant and Chepuri, Sundeep Prabhakar}
}

We focus on graph classification using a graph neural network (GNN) model that precomputes the node features using a bank of neighborhood aggregation graph operators arranged in parallel. These GNN models have a natural advantage of reduced training and inference time due to the precomputations but are also fundamentally different from popular GNN variants that update node features through a sequential neighborhood aggregation procedure during training. We provide theoretical conditions under which a generic GNN model with parallel neighborhood aggregations (PA-GNNs, in short) are provably as powerful as the well-known Weisfeiler-Lehman (WL) graph isomorphism test in discriminating non-isomorphic graphs. Although PA-GNN models do not have an apparent relationship with the WL test, we show that the graph embeddings obtained from these two methods are injectively related. We then propose a specialized PA-GNN model, called SPIN, which obeys the developed conditions. We demonstrate via numerical experiments that the developed model achieves state-of-the-art performance on many diverse real-world datasets while maintaining the discriminative power of the WL test and the computational advantage of preprocessing graphs before the training process.

@article{
 title = {HoloCast+: Hybrid Digital-Analog Transmission for Graceful Point Cloud Delivery with Graph Fourier Transform},
 type = {article},
 year = {2021},
 keywords = {Decorrelation,Distortion,Encoding,Graph Signal Processing,Hybrid Digital-Analog Coding,Image reconstruction,Point Cloud,Receivers,Three-dimensional displays,Wireless Transmission,Wireless communication},
 pages = {1-13},
 id = {25b68138-b0b3-3dd0-97da-f19236401608},
 created = {2022-03-02T07:02:50.312Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-02T07:03:02.516Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {Point cloud is an emerging data format useful for various applications such has holographic display, autonomous vehicle, and augmented reality. Conventionally, communications of point cloud data have relied on digital compression and digital modulation for three-dimensional (3D) data streaming. However, such digital-based delivery schemes have fundamental issues called cliff and leveling effects, where the 3D reconstruction quality is a step function in terms of wireless channel quality. We propose a novel scheme of point cloud delivery, called HoloCast+, to overcome cliff and leveling effects. Specifically, our method utilizes hybrid digital-analog coding, integrating digital compression and analog coding based on graph Fourier transform (GFT), to gracefully improve 3D reconstruction quality with the improvement of channel quality. We demonstrate that HoloCast+ offers better 3D reconstruction quality in terms of the symmetric mean square error (sMSE) by up to 18.3 dB and 10.5 dB, respectively, compared to conventional digital-based and analog-based delivery methods in wireless fading environments.},
 bibtype = {article},
 author = {Fujihashi, Takuya and Koike-Akino, Toshiaki and Watanabe, Takashi and Orlik, Philip V.},
 doi = {10.1109/TMM.2021.3077772},
 journal = {IEEE Transactions on Multimedia}
}

Point cloud is an emerging data format useful for various applications such has holographic display, autonomous vehicle, and augmented reality. Conventionally, communications of point cloud data have relied on digital compression and digital modulation for three-dimensional (3D) data streaming. However, such digital-based delivery schemes have fundamental issues called cliff and leveling effects, where the 3D reconstruction quality is a step function in terms of wireless channel quality. We propose a novel scheme of point cloud delivery, called HoloCast+, to overcome cliff and leveling effects. Specifically, our method utilizes hybrid digital-analog coding, integrating digital compression and analog coding based on graph Fourier transform (GFT), to gracefully improve 3D reconstruction quality with the improvement of channel quality. We demonstrate that HoloCast+ offers better 3D reconstruction quality in terms of the symmetric mean square error (sMSE) by up to 18.3 dB and 10.5 dB, respectively, compared to conventional digital-based and analog-based delivery methods in wireless fading environments.

@article{
 title = {Advanced Geometry Surface Coding for Dynamic Point Cloud Compression},
 type = {article},
 year = {2021},
 pages = {1-10},
 websites = {http://arxiv.org/abs/2103.06549},
 id = {e8c252bb-c036-35a0-9a97-5f5f9c955ad0},
 created = {2022-03-02T07:02:50.313Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-02T09:31:16.710Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {In video-based dynamic point cloud compression (V-PCC), 3D point clouds are projected onto 2D images for compressing with the existing video codecs. However, the existing video codecs are originally designed for natural visual signals, and it fails to account for the characteristics of point clouds. Thus, there are still problems in the compression of geometry information generated from the point clouds. Firstly, the distortion model in the existing rate-distortion optimization (RDO) is not consistent with the geometry quality assessment metrics. Secondly, the prediction methods in video codecs fail to account for the fact that the highest depth values of a far layer is greater than or equal to the corresponding lowest depth values of a near layer. This paper proposes an advanced geometry surface coding (AGSC) method for dynamic point clouds (DPC) compression. The proposed method consists of two modules, including an error projection model-based (EPM-based) RDO and an occupancy map-based (OM-based) merge prediction. Firstly, the EPM model is proposed to describe the relationship between the distortion model in the existing video codec and the geometry quality metric. Secondly, the EPM-based RDO method is presented to project the existing distortion model on the plane normal and is simplified to estimate the average normal vectors of coding units (CUs). Finally, we propose the OM-based merge prediction approach, in which the prediction pixels of merge modes are refined based on the occupancy map. Experiments tested on the standard point clouds show that the proposed method achieves an average 9.84\% bitrate saving for geometry compression.},
 bibtype = {article},
 author = {Xiong, Jian and Gao, Hao and Wang, Miaohui and Li, Hongliang and Ngan, King Ngi and Lin, Weisi}
}

In video-based dynamic point cloud compression (V-PCC), 3D point clouds are projected onto 2D images for compressing with the existing video codecs. However, the existing video codecs are originally designed for natural visual signals, and it fails to account for the characteristics of point clouds. Thus, there are still problems in the compression of geometry information generated from the point clouds. Firstly, the distortion model in the existing rate-distortion optimization (RDO) is not consistent with the geometry quality assessment metrics. Secondly, the prediction methods in video codecs fail to account for the fact that the highest depth values of a far layer is greater than or equal to the corresponding lowest depth values of a near layer. This paper proposes an advanced geometry surface coding (AGSC) method for dynamic point clouds (DPC) compression. The proposed method consists of two modules, including an error projection model-based (EPM-based) RDO and an occupancy map-based (OM-based) merge prediction. Firstly, the EPM model is proposed to describe the relationship between the distortion model in the existing video codec and the geometry quality metric. Secondly, the EPM-based RDO method is presented to project the existing distortion model on the plane normal and is simplified to estimate the average normal vectors of coding units (CUs). Finally, we propose the OM-based merge prediction approach, in which the prediction pixels of merge modes are refined based on the occupancy map. Experiments tested on the standard point clouds show that the proposed method achieves an average 9.84\% bitrate saving for geometry compression.

@article{
 title = {AR Anchor System Using Mobile Based 3D GNN Detection},
 type = {article},
 year = {2021},
 keywords = {3d object detection,augmented reality,deep learning,gnn,point cloud,smart phone},
 pages = {54-60},
 volume = {13},
 id = {db9ff35d-c764-3fc8-ba97-efd5fbfcbcff},
 created = {2022-03-02T07:52:11.253Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T09:33:30.101Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {AR (Augmented Reality) is a technology that provides virtual content to the real world and provides additional information to objects in real-time through 3D content. In the past, a high-performance device was required to experience AR, but it was possible to implement AR more easily by improving mobile performance and mounting various sensors such as ToF (Time-of-Flight). Also, the importance of mobile augmented reality is growing with the commercialization of high-speed wireless Internet such as 5G. Thus, this paper proposes a system that can provide AR services via GNN (Graph Neural Network) using cameras and sensors on mobile devices. ToF of mobile devices is used to capture depth maps. A 3D point cloud was created using RGB images to distinguish specific colors of objects. Point clouds created with RGB images and Depth Map perform downsampling for smooth communication between mobile and server. Point clouds sent to the server are used for 3D object detection. The detection process determines the class of objects and uses one point in the 3D bounding box as an anchor point. AR contents are provided through app and web through class and anchor of the detected object. PU  - The Institute of Internet, Broadcasting and Communication},
 bibtype = {article},
 author = {University, Graduate School of Smart Convergence， Kwangwoon and Kim, Jun-Sik and Kim, Dong-Kyun and CHUL, KWON SOON and Jung, Kye-Dong},
 journal = {The International Journal of Internet, Broadcasting and Communication},
 number = {1}
}

AR (Augmented Reality) is a technology that provides virtual content to the real world and provides additional information to objects in real-time through 3D content. In the past, a high-performance device was required to experience AR, but it was possible to implement AR more easily by improving mobile performance and mounting various sensors such as ToF (Time-of-Flight). Also, the importance of mobile augmented reality is growing with the commercialization of high-speed wireless Internet such as 5G. Thus, this paper proposes a system that can provide AR services via GNN (Graph Neural Network) using cameras and sensors on mobile devices. ToF of mobile devices is used to capture depth maps. A 3D point cloud was created using RGB images to distinguish specific colors of objects. Point clouds created with RGB images and Depth Map perform downsampling for smooth communication between mobile and server. Point clouds sent to the server are used for 3D object detection. The detection process determines the class of objects and uses one point in the 3D bounding box as an anchor point. AR contents are provided through app and web through class and anchor of the detected object. PU - The Institute of Internet, Broadcasting and Communication

@article{
 title = {Branchy-GNN: A device-edge co-inference framework for efficient point cloud processing},
 type = {article},
 year = {2021},
 keywords = {Edge inference,Graph neural network (GNN),Joint source-channel coding (JSCC),Point cloud},
 pages = {8488-8492},
 volume = {2021-June},
 id = {0f1c9ca4-08dc-3efa-a7df-14a4986f90f9},
 created = {2022-03-02T07:52:11.254Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T09:33:30.114Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {The recent advancements of three-dimensional (3D) data acquisition devices have spurred a new breed of applications that rely on point cloud data processing. However, processing a large volume of point cloud data brings a significant workload on resource-constrained mobile devices, prohibiting from unleashing their full potentials. Built upon the emerging paradigm of device-edge co-inference, where an edge device extracts and transmits the intermediate feature to an edge server for further processing, we propose Branchy-GNN for efficient graph neural network (GNN) based point cloud processing by leveraging edge computing platforms. In order to reduce the on-device computational cost, the Branchy-GNN adds branch networks for early exiting. Besides, it employs learning-based joint source-channel coding (JSCC) for the intermediate feature compression to reduce the communication overhead. Our experimental results demonstrate that the proposed Branchy-GNN secures a significant latency reduction compared with several benchmark methods.},
 bibtype = {article},
 author = {Shao, Jiawei and Zhang, Haowei and Mao, Yuyi and Zhang, Jun},
 doi = {10.1109/ICASSP39728.2021.9414831},
 journal = {ICASSP, IEEE International Conference on Acoustics, Speech and Signal Processing - Proceedings}
}

The recent advancements of three-dimensional (3D) data acquisition devices have spurred a new breed of applications that rely on point cloud data processing. However, processing a large volume of point cloud data brings a significant workload on resource-constrained mobile devices, prohibiting from unleashing their full potentials. Built upon the emerging paradigm of device-edge co-inference, where an edge device extracts and transmits the intermediate feature to an edge server for further processing, we propose Branchy-GNN for efficient graph neural network (GNN) based point cloud processing by leveraging edge computing platforms. In order to reduce the on-device computational cost, the Branchy-GNN adds branch networks for early exiting. Besides, it employs learning-based joint source-channel coding (JSCC) for the intermediate feature compression to reduce the communication overhead. Our experimental results demonstrate that the proposed Branchy-GNN secures a significant latency reduction compared with several benchmark methods.

@article{
 title = {Split Computing and Early Exiting for Deep Learning Applications: Survey and Research Challenges},
 type = {article},
 year = {2021},
 keywords = {Split Computing, Edge Computing, Early Exit, Neura},
 volume = {37},
 websites = {http://arxiv.org/abs/2103.04505},
 publisher = {Association for Computing Machinery},
 id = {9642a144-c4ee-3757-a6c7-5d5769144cad},
 created = {2022-03-09T07:18:42.318Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-06-11T00:43:43.376Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449,a6e140dd-a959-4148-9ed8-b29d0c7966c6},
 private_publication = {false},
 abstract = {Mobile devices such as smartphones and autonomous vehicles increasingly rely on deep neural networks (DNNs) to execute complex inference tasks such as image classification and speech recognition, among others. However, continuously executing the entire DNN on the mobile device can quickly deplete its battery. Although task offloading to cloud/edge servers may decrease the mobile device's computational burden, erratic patterns in channel quality, network, and edge server load can lead to a significant delay in task execution. Recently, approaches based on split computing (SC) have been proposed, where the DNN is split into a head and a tail model, executed respectively on the mobile device and on the edge server. Ultimately, this may reduce bandwidth usage as well as energy consumption. Another approach, called early exiting (EE), trains models to present multiple "exits" earlier in the architecture, each providing increasingly higher target accuracy. Therefore, the trade-off between accuracy and delay can be tuned according to the current conditions or application demands. In this paper, we provide a comprehensive survey of the state of the art in SC and EE strategies by presenting a comparison of the most relevant approaches. We conclude the paper by providing a set of compelling research challenges.},
 bibtype = {article},
 author = {Matsubara, Yoshitomo and Levorato, Marco and Restuccia, Francesco},
 journal = {ACM Computing Surveys},
 number = {4}
}

Mobile devices such as smartphones and autonomous vehicles increasingly rely on deep neural networks (DNNs) to execute complex inference tasks such as image classification and speech recognition, among others. However, continuously executing the entire DNN on the mobile device can quickly deplete its battery. Although task offloading to cloud/edge servers may decrease the mobile device's computational burden, erratic patterns in channel quality, network, and edge server load can lead to a significant delay in task execution. Recently, approaches based on split computing (SC) have been proposed, where the DNN is split into a head and a tail model, executed respectively on the mobile device and on the edge server. Ultimately, this may reduce bandwidth usage as well as energy consumption. Another approach, called early exiting (EE), trains models to present multiple "exits" earlier in the architecture, each providing increasingly higher target accuracy. Therefore, the trade-off between accuracy and delay can be tuned according to the current conditions or application demands. In this paper, we provide a comprehensive survey of the state of the art in SC and EE strategies by presenting a comparison of the most relevant approaches. We conclude the paper by providing a set of compelling research challenges.

@article{
 title = {Superpoint-guided Semi-supervised Semantic Segmentation of 3D Point Clouds},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.03601},
 id = {bb061536-5750-3d90-9594-45493dcdca2d},
 created = {2022-03-09T09:35:04.531Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T07:29:33.498Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5},
 private_publication = {false},
 abstract = {3D point cloud semantic segmentation is a challenging topic in the computer vision field. Most of the existing methods in literature require a large amount of fully labeled training data, but it is extremely time-consuming to obtain these training data by manually labeling massive point clouds. Addressing this problem, we propose a superpoint-guided semi-supervised segmentation network for 3D point clouds, which jointly utilizes a small portion of labeled scene point clouds and a large number of unlabeled point clouds for network training. The proposed network is iteratively updated with its predicted pseudo labels, where a superpoint generation module is introduced for extracting superpoints from 3D point clouds, and a pseudo-label optimization module is explored for automatically assigning pseudo labels to the unlabeled points under the constraint of the extracted superpoints. Additionally, there are some 3D points without pseudo-label supervision. We propose an edge prediction module to constrain features of edge points. A superpoint feature aggregation module and a superpoint feature consistency loss function are introduced to smooth superpoint features. Extensive experimental results on two 3D public datasets demonstrate that our method can achieve better performance than several state-of-the-art point cloud segmentation networks and several popular semi-supervised segmentation methods with few labeled scenes.},
 bibtype = {article},
 author = {Deng, Shuang and Dong, Qiulei and Liu, Bo and Hu, Zhanyi}
}

3D point cloud semantic segmentation is a challenging topic in the computer vision field. Most of the existing methods in literature require a large amount of fully labeled training data, but it is extremely time-consuming to obtain these training data by manually labeling massive point clouds. Addressing this problem, we propose a superpoint-guided semi-supervised segmentation network for 3D point clouds, which jointly utilizes a small portion of labeled scene point clouds and a large number of unlabeled point clouds for network training. The proposed network is iteratively updated with its predicted pseudo labels, where a superpoint generation module is introduced for extracting superpoints from 3D point clouds, and a pseudo-label optimization module is explored for automatically assigning pseudo labels to the unlabeled points under the constraint of the extracted superpoints. Additionally, there are some 3D points without pseudo-label supervision. We propose an edge prediction module to constrain features of edge points. A superpoint feature aggregation module and a superpoint feature consistency loss function are introduced to smooth superpoint features. Extensive experimental results on two 3D public datasets demonstrate that our method can achieve better performance than several state-of-the-art point cloud segmentation networks and several popular semi-supervised segmentation methods with few labeled scenes.

@article{
 title = {AirNet: Neural Network Transmission over the Air},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2105.11166},
 id = {c0b9ce3f-b1ee-370b-943c-745e9f0d6dcf},
 created = {2022-03-09T09:35:04.532Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-09T09:35:12.876Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {84eaadea-8864-4baf-9a7a-b5a2f5b96449},
 private_publication = {false},
 abstract = {State-of-the-art performance for many emerging edge applications is achieved by deep neural networks (DNNs). Often, these DNNs are location and time sensitive, and the parameters of a specific DNN must be delivered from an edge server to the edge device rapidly and efficiently to carry out time-sensitive inference tasks. We introduce AirNet, a novel training and analog transmission method that allows efficient wireless delivery of DNNs. We first train the DNN with noise injection to counter the wireless channel noise. We also employ pruning to reduce the channel bandwidth necessary for transmission, and perform knowledge distillation from a larger model to achieve satisfactory performance, despite the channel perturbations. We show that AirNet achieves significantly higher test accuracy compared to digital alternatives under the same bandwidth and power constraints. It also exhibits graceful degradation with channel quality, which reduces the requirement for accurate channel estimation.},
 bibtype = {article},
 author = {Jankowski, Mikolaj and Gunduz, Deniz and Mikolajczyk, Krystian},
 number = {677854}
}

State-of-the-art performance for many emerging edge applications is achieved by deep neural networks (DNNs). Often, these DNNs are location and time sensitive, and the parameters of a specific DNN must be delivered from an edge server to the edge device rapidly and efficiently to carry out time-sensitive inference tasks. We introduce AirNet, a novel training and analog transmission method that allows efficient wireless delivery of DNNs. We first train the DNN with noise injection to counter the wireless channel noise. We also employ pruning to reduce the channel bandwidth necessary for transmission, and perform knowledge distillation from a larger model to achieve satisfactory performance, despite the channel perturbations. We show that AirNet achieves significantly higher test accuracy compared to digital alternatives under the same bandwidth and power constraints. It also exhibits graceful degradation with channel quality, which reduces the requirement for accurate channel estimation.

@article{
 title = {Point Cloud Instance Segmentation of Indoor Scenes Using Learned Pairwise Patch Relations},
 type = {article},
 year = {2021},
 keywords = {Point clouds,classifier,indoor scenes,instance segmentation,patch relations},
 pages = {15891-15901},
 volume = {9},
 id = {fb50a7a8-2eb9-31aa-8157-74dfa9015592},
 created = {2022-03-18T10:02:57.315Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-18T10:03:05.478Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5},
 private_publication = {false},
 abstract = {Indoor scene understanding is an active research topic that has recently gained increasing attention in both computer graphics and computer vision community. This paper presents a novel automatic point cloud instance abstraction and segmentation of indoor scenes using learned pairwise planar patch relations. Planar patches have sufficient representative power to abstract the semantics in mostly man-made 3D indoor scenes. To exploit the planar patch relationships, a new indoor point clouds dataset is automatically derived from the public dataset ScanNet, which can be adopted for training a pairwise patch relation classifier. Given the pre-processed indoor scene point clouds, a set of planar patches are generated to provide robust and compact representation of indoor objects and parts. For each extracted planar patch, a feature descriptor in view of its geometric properties and appearance is calculated. Owing to the proposed feature descriptors, the relations of pairwise planar patch are measured and can be fed into the pre-trained patch relation classifier. To yield the instance segmentation of indoor scenes, a robust graph-based algorithm is adopted to group these planar patches which incorporate the learned patch relations. Experimental results demonstrate that our proposed approach can produce instance semantic segmentation of various cluttered indoor scenes robustly and efficiently.},
 bibtype = {article},
 author = {Yu, Lijie and Sun, Yuliang and Zhang, Xudong and Miao, Yongwei and Zhang, Xiuli},
 doi = {10.1109/ACCESS.2021.3051618},
 journal = {IEEE Access}
}

Indoor scene understanding is an active research topic that has recently gained increasing attention in both computer graphics and computer vision community. This paper presents a novel automatic point cloud instance abstraction and segmentation of indoor scenes using learned pairwise planar patch relations. Planar patches have sufficient representative power to abstract the semantics in mostly man-made 3D indoor scenes. To exploit the planar patch relationships, a new indoor point clouds dataset is automatically derived from the public dataset ScanNet, which can be adopted for training a pairwise patch relation classifier. Given the pre-processed indoor scene point clouds, a set of planar patches are generated to provide robust and compact representation of indoor objects and parts. For each extracted planar patch, a feature descriptor in view of its geometric properties and appearance is calculated. Owing to the proposed feature descriptors, the relations of pairwise planar patch are measured and can be fed into the pre-trained patch relation classifier. To yield the instance segmentation of indoor scenes, a robust graph-based algorithm is adopted to group these planar patches which incorporate the learned patch relations. Experimental results demonstrate that our proposed approach can produce instance semantic segmentation of various cluttered indoor scenes robustly and efficiently.

@article{
 title = {Multi-scale Point Cloud Analysis},
 type = {article},
 year = {2021},
 id = {db3c2bfe-247b-33f1-a4b2-438adab69ed3},
 created = {2022-03-22T06:52:16.996Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-22T06:52:21.360Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5},
 private_publication = {false},
 bibtype = {article},
 author = {Lejemble, Thibault}
}

@article{
 title = {Facevae: Generation of a 3D geometric object using variational autoencoders},
 type = {article},
 year = {2021},
 keywords = {3D geometry,Deep learning,Generation model,Graph data,VAE},
 pages = {1-14},
 volume = {10},
 id = {25184c64-4522-35b7-8d56-a150264d7e9c},
 created = {2022-03-23T06:17:59.176Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:13.688Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Park2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {Deep learning for 3D data has become a popular research theme in many fields. However, most of the research on 3D data is based on voxels, 2D images, and point clouds. At actual industrial sites, face-based geometry data are being used, but their direct application to industrial sites remains limited due to a lack of existing research. In this study, to overcome these limitations, we present a face-based variational autoencoder (FVAE) model that generates 3D geometry data using a variational autoencoder (VAE) model directly from face-based geometric data. Our model improves the existing node and edge-based adjacency matrix and optimizes it for geometric learning by using a face-and edge-based adjacency matrix according to the 3D geometry structure. In the experiment, we achieved the result of generating adjacency matrix information with 72% precision and 69% recall through end-to-end learning of Face-Based 3D Geometry. In addition, we presented various structurization methods for 3D unstructured geometry and compared their performance, and proved the method to effectively perform reconstruction of the learned structured data through experiments.},
 bibtype = {article},
 author = {Park, Sungsoo and Kim, Hyeoncheol},
 doi = {10.3390/electronics10222792},
 journal = {Electronics (Switzerland)},
 number = {22}
}

Deep learning for 3D data has become a popular research theme in many fields. However, most of the research on 3D data is based on voxels, 2D images, and point clouds. At actual industrial sites, face-based geometry data are being used, but their direct application to industrial sites remains limited due to a lack of existing research. In this study, to overcome these limitations, we present a face-based variational autoencoder (FVAE) model that generates 3D geometry data using a variational autoencoder (VAE) model directly from face-based geometric data. Our model improves the existing node and edge-based adjacency matrix and optimizes it for geometric learning by using a face-and edge-based adjacency matrix according to the 3D geometry structure. In the experiment, we achieved the result of generating adjacency matrix information with 72% precision and 69% recall through end-to-end learning of Face-Based 3D Geometry. In addition, we presented various structurization methods for 3D unstructured geometry and compared their performance, and proved the method to effectively perform reconstruction of the learned structured data through experiments.

@article{
 title = {Hierarchical Graph-Convolutional Variational AutoEncoding for Generative Modelling of Human Motion},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.12602},
 id = {53726edd-7ad6-31b2-b958-9d5e223d7f8f},
 created = {2022-03-23T06:17:59.177Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:13.113Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Bourached2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {Models of human motion commonly focus either on trajectory prediction or action classification but rarely both. The marked heterogeneity and intricate compositionality of human motion render each task vulnerable to the data degradation and distributional shift common to real-world scenarios. A sufficiently expressive generative model of action could in theory enable data conditioning and distributional resilience within a unified framework applicable to both tasks. Here we propose a novel architecture based on hierarchical variational autoencoders and deep graph convolutional neural networks for generating a holistic model of action over multiple time-scales. We show this Hierarchical Graph-convolutional Variational Autoencoder (HG-VAE) to be capable of generating coherent actions, detecting out-of-distribution data, and imputing missing data by gradient ascent on the model's posterior. Trained and evaluated on H3.6M and the largest collection of open source human motion data, AMASS, we show HG-VAE can facilitate downstream discriminative learning better than baseline models.},
 bibtype = {article},
 author = {Bourached, Anthony and Gray, Robert and Griffiths, Ryan-Rhys and Jha, Ashwani and Nachev, Parashkev}
}

Models of human motion commonly focus either on trajectory prediction or action classification but rarely both. The marked heterogeneity and intricate compositionality of human motion render each task vulnerable to the data degradation and distributional shift common to real-world scenarios. A sufficiently expressive generative model of action could in theory enable data conditioning and distributional resilience within a unified framework applicable to both tasks. Here we propose a novel architecture based on hierarchical variational autoencoders and deep graph convolutional neural networks for generating a holistic model of action over multiple time-scales. We show this Hierarchical Graph-convolutional Variational Autoencoder (HG-VAE) to be capable of generating coherent actions, detecting out-of-distribution data, and imputing missing data by gradient ascent on the model's posterior. Trained and evaluated on H3.6M and the largest collection of open source human motion data, AMASS, we show HG-VAE can facilitate downstream discriminative learning better than baseline models.

@article{
 title = {A Two-Stage Cascade Model with Variational Autoencoders and Attention Gates for MRI Brain Tumor Segmentation},
 type = {article},
 year = {2021},
 keywords = {Attention gate,Brain tumor segmentation,Encoder-decoder network,Variational autoencoder},
 pages = {435-447},
 volume = {12658 LNCS},
 id = {c849ae67-71d8-3a07-b804-07d823146590},
 created = {2022-03-23T06:17:59.178Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:12.522Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Lyu2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {Automatic MRI brain tumor segmentation is of vital importance for the disease diagnosis, monitoring, and treatment planning. In this paper, we propose a two-stage encoder-decoder based model for brain tumor subregional segmentation. Variational autoencoder regularization is utilized in both stages to prevent the overfitting issue. The second-stage network adopts attention gates and is trained additionally using an expanded dataset formed by the first-stage outputs. On the BraTS 2020 validation dataset, the proposed method achieves the mean Dice score of 0.9041, 0.8350, and 0.7958, and Hausdorff distance (95%) of 4.953, 6.299, 23.608 for the whole tumor, tumor core, and enhancing tumor, respectively. The corresponding results on the BraTS 2020 testing dataset are 0.8729, 0.8357, and 0.8205 for Dice score, and 11.4288, 19.9690, and 15.6711 for Hausdorff distance. The code is publicly available at https://github.com/shu-hai/two-stage-VAE-Attention-gate-BraTS2020.},
 bibtype = {article},
 author = {Lyu, Chenggang and Shu, Hai},
 doi = {10.1007/978-3-030-72084-1_39},
 journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)}
}

Automatic MRI brain tumor segmentation is of vital importance for the disease diagnosis, monitoring, and treatment planning. In this paper, we propose a two-stage encoder-decoder based model for brain tumor subregional segmentation. Variational autoencoder regularization is utilized in both stages to prevent the overfitting issue. The second-stage network adopts attention gates and is trained additionally using an expanded dataset formed by the first-stage outputs. On the BraTS 2020 validation dataset, the proposed method achieves the mean Dice score of 0.9041, 0.8350, and 0.7958, and Hausdorff distance (95%) of 4.953, 6.299, 23.608 for the whole tumor, tumor core, and enhancing tumor, respectively. The corresponding results on the BraTS 2020 testing dataset are 0.8729, 0.8357, and 0.8205 for Dice score, and 11.4288, 19.9690, and 15.6711 for Hausdorff distance. The code is publicly available at https://github.com/shu-hai/two-stage-VAE-Attention-gate-BraTS2020.

@article{
 title = {NeRF-VAE: A Geometry Aware 3D Scene Generative Model},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.00587},
 id = {2a6ac3d1-9a73-396e-a7da-c9e670768a81},
 created = {2022-03-23T06:17:59.184Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:13.506Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Kosiorek2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {We propose NeRF-VAE, a 3D scene generative model that incorporates geometric structure via NeRF and differentiable volume rendering. In contrast to NeRF, our model takes into account shared structure across scenes, and is able to infer the structure of a novel scene -- without the need to re-train -- using amortized inference. NeRF-VAE's explicit 3D rendering process further contrasts previous generative models with convolution-based rendering which lacks geometric structure. Our model is a VAE that learns a distribution over radiance fields by conditioning them on a latent scene representation. We show that, once trained, NeRF-VAE is able to infer and render geometrically-consistent scenes from previously unseen 3D environments using very few input images. We further demonstrate that NeRF-VAE generalizes well to out-of-distribution cameras, while convolutional models do not. Finally, we introduce and study an attention-based conditioning mechanism of NeRF-VAE's decoder, which improves model performance.},
 bibtype = {article},
 author = {Kosiorek, Adam R. and Strathmann, Heiko and Zoran, Daniel and Moreno, Pol and Schneider, Rosalia and Mokrá, Soňa and Rezende, Danilo J.}
}

We propose NeRF-VAE, a 3D scene generative model that incorporates geometric structure via NeRF and differentiable volume rendering. In contrast to NeRF, our model takes into account shared structure across scenes, and is able to infer the structure of a novel scene -- without the need to re-train -- using amortized inference. NeRF-VAE's explicit 3D rendering process further contrasts previous generative models with convolution-based rendering which lacks geometric structure. Our model is a VAE that learns a distribution over radiance fields by conditioning them on a latent scene representation. We show that, once trained, NeRF-VAE is able to infer and render geometrically-consistent scenes from previously unseen 3D environments using very few input images. We further demonstrate that NeRF-VAE generalizes well to out-of-distribution cameras, while convolutional models do not. Finally, we introduce and study an attention-based conditioning mechanism of NeRF-VAE's decoder, which improves model performance.

@article{
 title = {A Unified 3D Human Motion Synthesis Model via Conditional Variational Auto-Encoder *},
 type = {article},
 year = {2021},
 pages = {11645-11655},
 id = {9e6f497a-b560-3fc8-ad30-9d9ba7820bbf},
 created = {2022-03-23T06:17:59.205Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:12.917Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Cai2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {We present a unified and flexible framework to address the generalized problem of 3D motion synthesis that covers the tasks of motion prediction, completion, interpolation , and spatial-temporal recovery. Since these tasks have different input constraints and various fidelity and diversity requirements, most existing approaches only cater to a specific task or use different architectures to address various tasks. Here we propose a unified framework based on Conditional Variational Auto-Encoder (CVAE), where we treat any arbitrary input as a masked motion series. Notably, by considering this problem as a conditional generation process , we estimate a parametric distribution of the missing regions based on the input conditions, from which to sample and synthesize the full motion series. To further allow the flexibility of manipulating the motion style of the generated series, we design an Action-Adaptive Modulation (AAM) to propagate the given semantic guidance through the whole sequence. We also introduce a cross-attention mechanism to exploit distant relations among decoder and encoder features for better realism and global consistency. We conducted extensive experiments on Human 3.6M and CMU-Mocap. The results show that our method produces coherent and realistic results for various motion synthesis tasks, *},
 bibtype = {article},
 author = {Cai, Yujun and Wang, Yiwei and Zhu, Yiheng and Cham, Tat-Jen and Cai, Jianfei and Yuan, Junsong and Liu, Jun and Zheng, Chuanxia and Yan, Sijie and Ding, Henghui and Shen, Xiaohui and Liu, Ding and Thalmann, Nadia Magnenat},
 journal = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}
}

We present a unified and flexible framework to address the generalized problem of 3D motion synthesis that covers the tasks of motion prediction, completion, interpolation , and spatial-temporal recovery. Since these tasks have different input constraints and various fidelity and diversity requirements, most existing approaches only cater to a specific task or use different architectures to address various tasks. Here we propose a unified framework based on Conditional Variational Auto-Encoder (CVAE), where we treat any arbitrary input as a masked motion series. Notably, by considering this problem as a conditional generation process , we estimate a parametric distribution of the missing regions based on the input conditions, from which to sample and synthesize the full motion series. To further allow the flexibility of manipulating the motion style of the generated series, we design an Action-Adaptive Modulation (AAM) to propagate the given semantic guidance through the whole sequence. We also introduce a cross-attention mechanism to exploit distant relations among decoder and encoder features for better realism and global consistency. We conducted extensive experiments on Human 3.6M and CMU-Mocap. The results show that our method produces coherent and realistic results for various motion synthesis tasks, *

@article{
 title = {EditVAE: Unsupervised Part-Aware Controllable 3D Point Cloud Shape Generation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2110.06679},
 id = {c0f68cca-5451-3d90-b18f-663e89a38896},
 created = {2022-03-23T06:17:59.286Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-21T08:31:58.519Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Li2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {This paper tackles the problem of parts-aware point cloud generation. Unlike existing works which require the point cloud to be segmented into parts a priori, our parts-aware editing and generation is performed in an unsupervised manner. We achieve this with a simple modification of the Variational Auto-Encoder which yields a joint model of the point cloud itself along with a schematic representation of it as a combination of shape primitives. In particular, we introduce a latent representation of the point cloud which can be decomposed into a disentangled representation for each part of the shape. These parts are in turn disentangled into both a shape primitive and a point cloud representation, along with a standardising transformation to a canonical coordinate system. The dependencies between our standardising transformations preserve the spatial dependencies between the parts in a manner which allows meaningful parts-aware point cloud generation and shape editing. In addition to the flexibility afforded by our disentangled representation, the inductive bias introduced by our joint modelling approach yields the state-of-the-art experimental results on the ShapeNet dataset.},
 bibtype = {article},
 author = {Li, Shidi and Liu, Miaomiao and Walder, Christian}
}

This paper tackles the problem of parts-aware point cloud generation. Unlike existing works which require the point cloud to be segmented into parts a priori, our parts-aware editing and generation is performed in an unsupervised manner. We achieve this with a simple modification of the Variational Auto-Encoder which yields a joint model of the point cloud itself along with a schematic representation of it as a combination of shape primitives. In particular, we introduce a latent representation of the point cloud which can be decomposed into a disentangled representation for each part of the shape. These parts are in turn disentangled into both a shape primitive and a point cloud representation, along with a standardising transformation to a canonical coordinate system. The dependencies between our standardising transformations preserve the spatial dependencies between the parts in a manner which allows meaningful parts-aware point cloud generation and shape editing. In addition to the flexibility afforded by our disentangled representation, the inductive bias introduced by our joint modelling approach yields the state-of-the-art experimental results on the ShapeNet dataset.

@article{
 title = {Automated building change detection with amodal completion of point clouds},
 type = {article},
 year = {2021},
 keywords = {Building information modeling,Change detection,Deep learning,Point cloud,Point cloud completion},
 pages = {103568},
 volume = {124},
 websites = {https://doi.org/10.1016/j.autcon.2021.103568},
 publisher = {Elsevier B.V.},
 id = {52dbb417-9e35-3c96-8767-d7bf9dcb8d8f},
 created = {2022-03-23T06:17:59.298Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-01T09:16:03.012Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Czerniawski2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {When updating digital models of existing buildings, changes in the built environment are detected by comparing outdated BIMs with captured point clouds representing current conditions. Here we show that point cloud completion (i.e. automated filling-in of missing data) improves the accuracy of change detection. We perform point cloud completion using a hierarchical deep variational autoencoder (a type of artificial neural network) modified to include skip connections between the convolution and deconvolution layers. The resulting receiver operating characteristic curve shows that completion boosts change detection performance from a total area under the curve of 0.55 to 0.75. Completion achieves this by eliminating differences between the BIM and point cloud inputs that are a consequence of incompleteness while distilling the differences due to building change. We anticipate that automated change detection methods with resilience to imperfect data will become more critical as automated building analyses become increasingly abstracted from data collection.},
 bibtype = {article},
 author = {Czerniawski, Thomas and Ma, Jong Won and Fernanda Leite, undefined},
 doi = {10.1016/j.autcon.2021.103568},
 journal = {Automation in Construction},
 number = {January}
}

When updating digital models of existing buildings, changes in the built environment are detected by comparing outdated BIMs with captured point clouds representing current conditions. Here we show that point cloud completion (i.e. automated filling-in of missing data) improves the accuracy of change detection. We perform point cloud completion using a hierarchical deep variational autoencoder (a type of artificial neural network) modified to include skip connections between the convolution and deconvolution layers. The resulting receiver operating characteristic curve shows that completion boosts change detection performance from a total area under the curve of 0.55 to 0.75. Completion achieves this by eliminating differences between the BIM and point cloud inputs that are a consequence of incompleteness while distilling the differences due to building change. We anticipate that automated change detection methods with resilience to imperfect data will become more critical as automated building analyses become increasingly abstracted from data collection.

@article{
 title = {Setvae: Learning hierarchical composition for generative modeling of set-structured data},
 type = {article},
 year = {2021},
 pages = {15054-15063},
 id = {7ac8b9c2-f0b5-3546-9927-03fd86136b31},
 created = {2022-03-23T06:17:59.333Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:11.879Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Kim2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {Generative modeling of set-structured data, such as point clouds, requires reasoning over local and global structures at various scales. However, adopting multi-scale frameworks for ordinary sequential data to a set-structured data is nontrivial as it should be invariant to the permutation of its elements. In this paper, we propose SetVAE, a hierarchical variational autoencoder for sets. Motivated by recent progress in set encoding, we build SetVAE upon attentive modules that first partition the set and project the partition back to the original cardinality. Exploiting this module, our hierarchical VAE learns latent variables at multiple scales, capturing coarse-to-fine dependency of the set elements while achieving permutation invariance. We evaluate our model on point cloud generation task and achieve competitive performance to the prior arts with substantially smaller model capacity. We qualitatively demonstrate that our model generalizes to unseen set sizes and learns interesting subset relations without supervision. Our implementation is available at https://github.com/jw9730/setvae.},
 bibtype = {article},
 author = {Kim, Jinwoo and Yoo, Jaehoon and Lee, Juho and Hong, Seunghoon},
 doi = {10.1109/CVPR46437.2021.01481},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

Generative modeling of set-structured data, such as point clouds, requires reasoning over local and global structures at various scales. However, adopting multi-scale frameworks for ordinary sequential data to a set-structured data is nontrivial as it should be invariant to the permutation of its elements. In this paper, we propose SetVAE, a hierarchical variational autoencoder for sets. Motivated by recent progress in set encoding, we build SetVAE upon attentive modules that first partition the set and project the partition back to the original cardinality. Exploiting this module, our hierarchical VAE learns latent variables at multiple scales, capturing coarse-to-fine dependency of the set elements while achieving permutation invariance. We evaluate our model on point cloud generation task and achieve competitive performance to the prior arts with substantially smaller model capacity. We qualitatively demonstrate that our model generalizes to unseen set sizes and learns interesting subset relations without supervision. Our implementation is available at https://github.com/jw9730/setvae.

@article{
 title = {Action-Conditioned 3D Human Motion Synthesis with Transformer VAE},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2104.05670},
 id = {640ac617-3379-352d-b4f3-0c15968cf81b},
 created = {2022-03-23T06:17:59.354Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-28T09:45:12.712Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Petrovich2021},
 folder_uuids = {1853f94b-7af1-40fa-b068-4758e9a02bc4,465973dc-f47a-4093-b987-b885254a351b},
 private_publication = {false},
 abstract = {We tackle the problem of action-conditioned generation of realistic and diverse human motion sequences. In contrast to methods that complete, or extend, motion sequences, this task does not require an initial pose or sequence. Here we learn an action-aware latent representation for human motions by training a generative variational autoencoder (VAE). By sampling from this latent space and querying a certain duration through a series of positional encodings, we synthesize variable-length motion sequences conditioned on a categorical action. Specifically, we design a Transformer-based architecture, ACTOR, for encoding and decoding a sequence of parametric SMPL human body models estimated from action recognition datasets. We evaluate our approach on the NTU RGB+D, HumanAct12 and UESTC datasets and show improvements over the state of the art. Furthermore, we present two use cases: improving action recognition through adding our synthesized data to training, and motion denoising. Code and models are available on our project page.},
 bibtype = {article},
 author = {Petrovich, Mathis and Black, Michael J. and Varol, Gül},
 doi = {10.1109/iccv48922.2021.01080}
}

We tackle the problem of action-conditioned generation of realistic and diverse human motion sequences. In contrast to methods that complete, or extend, motion sequences, this task does not require an initial pose or sequence. Here we learn an action-aware latent representation for human motions by training a generative variational autoencoder (VAE). By sampling from this latent space and querying a certain duration through a series of positional encodings, we synthesize variable-length motion sequences conditioned on a categorical action. Specifically, we design a Transformer-based architecture, ACTOR, for encoding and decoding a sequence of parametric SMPL human body models estimated from action recognition datasets. We evaluate our approach on the NTU RGB+D, HumanAct12 and UESTC datasets and show improvements over the state of the art. Furthermore, we present two use cases: improving action recognition through adding our synthesized data to training, and motion denoising. Code and models are available on our project page.

@article{
 title = {A brief survey on RGB-D semantic segmentation using deep learning},
 type = {article},
 year = {2021},
 keywords = {Deep learning,RGB-D datasets,Semantic segmentation},
 pages = {102080},
 volume = {70},
 websites = {https://doi.org/10.1016/j.displa.2021.102080},
 publisher = {Elsevier B.V.},
 id = {9683bc04-7e79-3270-81b6-3a1da421f52a},
 created = {2022-03-24T08:01:31.367Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-24T08:01:45.062Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {1e7b477c-c241-48c3-a542-ad06e3d39dd5},
 private_publication = {false},
 abstract = {Semantic segmentation is referred to as a process of linking each pixel in an image to a class label. With this pragmatic technique, it is possible to recognize different objects in an RGB image based on the color and texture, and hence it becomes easier to evaluate. Recently, researchers could perform semantic segmentation pretty well in RGB images. However, the methods based on RGB image lack enough information to realize semantic segmentation of complex scenes. RGB-D semantic segmentation with depth information has been proved to achieve better segmentation results by a lot of experiments, but there is a lack of a comprehensive survey. In this paper, the main purpose is to offer a detailed review of RGB-D semantic segmentation according to the research progress in recent years. Specifically, recently updated RGB-D datasets will be focused on first, and problems on RGB-D semantic segmentation will be discussed. In the end, a comprehensive analysis is carried out on recent methods and their analysis of the semantic segmentation.},
 bibtype = {article},
 author = {Wang, Changshuo and Wang, Chen and Li, Weijun and Wang, Haining},
 doi = {10.1016/j.displa.2021.102080},
 journal = {Displays},
 number = {September}
}

Semantic segmentation is referred to as a process of linking each pixel in an image to a class label. With this pragmatic technique, it is possible to recognize different objects in an RGB image based on the color and texture, and hence it becomes easier to evaluate. Recently, researchers could perform semantic segmentation pretty well in RGB images. However, the methods based on RGB image lack enough information to realize semantic segmentation of complex scenes. RGB-D semantic segmentation with depth information has been proved to achieve better segmentation results by a lot of experiments, but there is a lack of a comprehensive survey. In this paper, the main purpose is to offer a detailed review of RGB-D semantic segmentation according to the research progress in recent years. Specifically, recently updated RGB-D datasets will be focused on first, and problems on RGB-D semantic segmentation will be discussed. In the end, a comprehensive analysis is carried out on recent methods and their analysis of the semantic segmentation.

@article{
 title = {CoFiNet: Reliable Coarse-to-fine Correspondences for Robust Point Cloud Registration},
 type = {article},
 year = {2021},
 websites = {https://arxiv.org/abs/2110.14076v1},
 month = {10},
 day = {26},
 id = {48a3d272-6530-3ee9-a6f3-9edf8ca82ca2},
 created = {2022-03-28T07:17:57.661Z},
 accessed = {2022-03-28},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-31T06:33:43.585Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {a3e73416-fb6e-4382-ad74-08c4dd4c0e94},
 private_publication = {false},
 abstract = {We study the problem of extracting correspondences between a pair of point
clouds for registration. For correspondence retrieval, existing works benefit
from matching sparse keypoints detected from dense points but usually struggle
to guarantee their repeatability. To address this issue, we present CoFiNet -
Coarse-to-Fine Network which extracts hierarchical correspondences from coarse
to fine without keypoint detection. On a coarse scale and guided by a weighting
scheme, our model firstly learns to match down-sampled nodes whose vicinity
points share more overlap, which significantly shrinks the search space of a
consecutive stage. On a finer scale, node proposals are consecutively expanded
to patches that consist of groups of points together with associated
descriptors. Point correspondences are then refined from the overlap areas of
corresponding patches, by a density-adaptive matching module capable to deal
with varying point density. Extensive evaluation of CoFiNet on both indoor and
outdoor standard benchmarks shows our superiority over existing methods.
Especially on 3DLoMatch where point clouds share less overlap, CoFiNet
significantly outperforms state-of-the-art approaches by at least 5% on
Registration Recall, with at most two-third of their parameters.},
 bibtype = {article},
 author = {Yu, Hao and Li, Fu and Saleh, Mahdi and Busam, Benjamin and Ilic, Slobodan},
 doi = {10.48550/arxiv.2110.14076}
}

We study the problem of extracting correspondences between a pair of point clouds for registration. For correspondence retrieval, existing works benefit from matching sparse keypoints detected from dense points but usually struggle to guarantee their repeatability. To address this issue, we present CoFiNet - Coarse-to-Fine Network which extracts hierarchical correspondences from coarse to fine without keypoint detection. On a coarse scale and guided by a weighting scheme, our model firstly learns to match down-sampled nodes whose vicinity points share more overlap, which significantly shrinks the search space of a consecutive stage. On a finer scale, node proposals are consecutively expanded to patches that consist of groups of points together with associated descriptors. Point correspondences are then refined from the overlap areas of corresponding patches, by a density-adaptive matching module capable to deal with varying point density. Extensive evaluation of CoFiNet on both indoor and outdoor standard benchmarks shows our superiority over existing methods. Especially on 3DLoMatch where point clouds share less overlap, CoFiNet significantly outperforms state-of-the-art approaches by at least 5% on Registration Recall, with at most two-third of their parameters.

@article{
 title = {Statistical Estimation of the Kullback–Leibler Divergence},
 type = {article},
 year = {2021},
 keywords = {Gaussian model,Kullback–Leibler divergence,Shannon differential entropy,asymptotic behavior,k-nearest neighbor statistics,mixtures,statistical estimates},
 pages = {544},
 volume = {9},
 websites = {https://www.mdpi.com/2227-7390/9/5/544},
 month = {1},
 id = {c2c3961f-21c1-3e9a-bfd3-1f8e9a133643},
 created = {2022-03-28T09:45:01.047Z},
 accessed = {2021-09-14},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:01:13.434Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {bulinskiStatisticalEstimationKullback2021},
 source_type = {article},
 notes = {Number: 5<br/>Publisher: Multidisciplinary Digital Publishing Institute},
 private_publication = {false},
 abstract = {Asymptotic unbiasedness and L2-consistency are established, under mild conditions, for the estimates of the Kullback\&ndash;Leibler divergence between two probability measures in Rd, absolutely continuous with respect to (w.r.t.) the Lebesgue measure. These estimates are based on certain k-nearest neighbor statistics for pair of independent identically distributed (i.i.d.) due vector samples. The novelty of results is also in treating mixture models. In particular, they cover mixtures of nondegenerate Gaussian measures. The mentioned asymptotic properties of related estimators for the Shannon entropy and cross-entropy are strengthened. Some applications are indicated.},
 bibtype = {article},
 author = {Bulinski, Alexander and Dimitrov, Denis},
 doi = {10.3390/math9050544},
 journal = {Mathematics},
 number = {5}
}

Asymptotic unbiasedness and L2-consistency are established, under mild conditions, for the estimates of the Kullback\–Leibler divergence between two probability measures in Rd, absolutely continuous with respect to (w.r.t.) the Lebesgue measure. These estimates are based on certain k-nearest neighbor statistics for pair of independent identically distributed (i.i.d.) due vector samples. The novelty of results is also in treating mixture models. In particular, they cover mixtures of nondegenerate Gaussian measures. The mentioned asymptotic properties of related estimators for the Shannon entropy and cross-entropy are strengthened. Some applications are indicated.

@article{
 title = {Knowledge Distillation and Student-Teacher Learning for Visual Intelligence: A Review and New Outlooks},
 type = {article},
 year = {2021},
 keywords = {Computational modeling,Deep neural networks (DNN),Knowledge distillation (KD),Knowledge transfer,Measurement,Speech recognition,Student-Teacher learning (S-T),Task analysis,Training,Visual intelligence.,Visualization},
 pages = {1},
 id = {321ac3b3-55e4-36ce-94c4-702bb2c6e7e4},
 created = {2022-03-28T09:45:01.256Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:01:26.749Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {wangKnowledgeDistillationStudentTeacher2021},
 source_type = {article},
 short_title = {Knowledge Distillation and Student-Teacher },
 notes = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
 private_publication = {false},
 abstract = {Deep neural models, in recent years, have been successful in almost every field. However, these models are huge, demanding heavy computation power. Besides, the performance boost is highly dependent on redundant labeled data. To achieve faster speeds and to handle the problems caused by the lack of labeled data, knowledge distillation (KD) has been proposed to transfer information learned from one model to another. KD is often characterized by the so-called ‘Student-Teacher’ (S-T) learning framework and has been broadly applied in model compression and knowledge transfer. This paper is about KD and S-T learning, which are being actively studied in recent years. First, we aim to provide explanations of what KD is and how/why it works. Then, we provide a comprehensive survey on the recent progress of KD methods together with S-T frameworks typically used for vision tasks. In general, we investigate some fundamental questions that have been driving this research area and thoroughly generalize the research progress and technical details. Additionally, we systematically analyze the research status of KD in vision applications. Finally, we discuss the potentials and open challenges of existing methods and prospect the future directions of KD and S-T learning.},
 bibtype = {article},
 author = {Wang, Lin and Yoon, Kuk-Jin},
 doi = {10.1109/TPAMI.2021.3055564},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}
}

Deep neural models, in recent years, have been successful in almost every field. However, these models are huge, demanding heavy computation power. Besides, the performance boost is highly dependent on redundant labeled data. To achieve faster speeds and to handle the problems caused by the lack of labeled data, knowledge distillation (KD) has been proposed to transfer information learned from one model to another. KD is often characterized by the so-called ‘Student-Teacher’ (S-T) learning framework and has been broadly applied in model compression and knowledge transfer. This paper is about KD and S-T learning, which are being actively studied in recent years. First, we aim to provide explanations of what KD is and how/why it works. Then, we provide a comprehensive survey on the recent progress of KD methods together with S-T frameworks typically used for vision tasks. In general, we investigate some fundamental questions that have been driving this research area and thoroughly generalize the research progress and technical details. Additionally, we systematically analyze the research status of KD in vision applications. Finally, we discuss the potentials and open challenges of existing methods and prospect the future directions of KD and S-T learning.

@article{
 title = {An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale},
 type = {article},
 year = {2021},
 keywords = {Computer Science - Artificial Intelligence,Computer Science - Computer Vision and Pattern Re,Computer Science - Machine Learning},
 websites = {http://arxiv.org/abs/2010.11929},
 month = {6},
 id = {ffa066fd-9bc0-3f2a-9a44-ceb9c3ce9d4f},
 created = {2022-03-28T09:45:01.398Z},
 accessed = {2022-03-27},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:01:34.686Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {dosovitskiyImageWorth16x162021},
 source_type = {article},
 short_title = {An Image is Worth 16x16 Words},
 notes = {arXiv: 2010.11929},
 private_publication = {false},
 abstract = {While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited. In vision, attention is either applied in conjunction with convolutional networks, or used to replace certain components of convolutional networks while keeping their overall structure in place. We show that this reliance on CNNs is not necessary and a pure transformer applied directly to sequences of image patches can perform very well on image classification tasks. When pre-trained on large amounts of data and transferred to multiple mid-sized or small image recognition benchmarks (ImageNet, CIFAR-100, VTAB, etc.), Vision Transformer (ViT) attains excellent results compared to state-of-the-art convolutional networks while requiring substantially fewer computational resources to train.},
 bibtype = {article},
 author = {Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and Uszkoreit, Jakob and Houlsby, Neil},
 journal = {arXiv:2010.11929 [cs]}
}

While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited. In vision, attention is either applied in conjunction with convolutional networks, or used to replace certain components of convolutional networks while keeping their overall structure in place. We show that this reliance on CNNs is not necessary and a pure transformer applied directly to sequences of image patches can perform very well on image classification tasks. When pre-trained on large amounts of data and transferred to multiple mid-sized or small image recognition benchmarks (ImageNet, CIFAR-100, VTAB, etc.), Vision Transformer (ViT) attains excellent results compared to state-of-the-art convolutional networks while requiring substantially fewer computational resources to train.

@inproceedings{
 title = {ToFNest: Efficient Normal Estimation for Time-of-Flight Depth Cameras},
 type = {inproceedings},
 year = {2021},
 pages = {1791-1798},
 websites = {https://openaccess.thecvf.com/content/ICCV2021W/ACVR/html/Molnar_ToFNest_Efficient_Normal_Estimation_for_Time-of-Flight_Depth_Cameras_ICCVW_2021_paper.html},
 id = {0c1069c6-961c-3920-9f15-8117f67a94e9},
 created = {2022-03-28T09:45:01.400Z},
 accessed = {2022-01-26},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:02:22.025Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {molnarToFNestEfficientNormal2021},
 source_type = {inproceedings},
 short_title = {ToFNest},
 private_publication = {false},
 bibtype = {inproceedings},
 author = {Molnár, Szilárd and Kelényi, Benjamin and Tamás, Levente}
}

@inproceedings{
 title = {Deep Optimized Priors for 3D Shape Modeling and Reconstruction},
 type = {inproceedings},
 year = {2021},
 pages = {3269-3278},
 websites = {https://openaccess.thecvf.com/content/CVPR2021/html/Yang_Deep_Optimized_Priors_for_3D_Shape_Modeling_and_Reconstruction_CVPR_2021_paper.html},
 id = {93a3ccc0-a72f-3ac3-9cc5-ecd3609e91ec},
 created = {2022-03-28T09:45:01.555Z},
 accessed = {2022-03-27},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:02:02.695Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {yangDeepOptimizedPriors2021},
 source_type = {inproceedings},
 private_publication = {false},
 bibtype = {inproceedings},
 author = {Yang, Mingyue and Wen, Yuxin and Chen, Weikai and Chen, Yongwei and Jia, Kui}
}

@article{
 title = {Feature Pyramid Network Based Efficient Normal Estimation and Filtering for Time-of-Flight Depth Cameras},
 type = {article},
 year = {2021},
 pages = {6257},
 volume = {21},
 month = {9},
 id = {b814c2ae-0e98-3fa6-a236-e73d5d250c4f},
 created = {2022-03-28T09:45:01.684Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-06-24T10:35:25.892Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {molnarFeaturePyramidNetwork2021},
 source_type = {article},
 private_publication = {false},
 abstract = {In this paper, an efficient normal estimation and filtering method for depth images acquired by Time-of-Flight (ToF) cameras is proposed. The method is based on a common feature pyramid networks (FPN) architecture. The normal estimation method is called ToFNest, and the filtering method ToFClean. Both of these low-level 3D point cloud processing methods start from the 2D depth images, projecting the measured data into the 3D space and computing a task-specific loss function. Despite the simplicity, the methods prove to be efficient in terms of robustness and runtime. In order to validate the methods, extensive evaluations on public and custom datasets were performed. Compared with the state-of-the-art methods, the ToFNest and ToFClean algorithms are faster by an order of magnitude without losing precision on public datasets.},
 bibtype = {article},
 author = {Molnár, Szilárd and Kelenyi, Benjamin and Tamas, Levente},
 doi = {10.3390/s21186257},
 journal = {Sensors}
}

In this paper, an efficient normal estimation and filtering method for depth images acquired by Time-of-Flight (ToF) cameras is proposed. The method is based on a common feature pyramid networks (FPN) architecture. The normal estimation method is called ToFNest, and the filtering method ToFClean. Both of these low-level 3D point cloud processing methods start from the 2D depth images, projecting the measured data into the 3D space and computing a task-specific loss function. Despite the simplicity, the methods prove to be efficient in terms of robustness and runtime. In order to validate the methods, extensive evaluations on public and custom datasets were performed. Compared with the state-of-the-art methods, the ToFNest and ToFClean algorithms are faster by an order of magnitude without losing precision on public datasets.

@article{
 title = {Plausible 3D Face Wrinkle Generation Using Variational Autoencoders},
 type = {article},
 year = {2021},
 pages = {1},
 websites = {https://www.computer.org/csdl/journal/tg/5555/01/09321747/1qmbmWE93ZS},
 month = {1},
 id = {8e665205-a603-3249-a9ae-93610b2b42fb},
 created = {2022-03-28T09:45:02.307Z},
 accessed = {2022-03-22},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:03:17.033Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {dengPlausible3DFace2021},
 source_type = {article},
 notes = {Publisher: IEEE Computer Society},
 private_publication = {false},
 abstract = {Realistic 3D facial modeling and animation have been increasingly used in many graphics, animation, and virtual reality applications. However, generating realistic fine-scale wrinkles on 3D faces, in particular, on animated 3D faces, is still a challenging problem that is far away from being resolved. In this paper we propose an end-to-end system to automatically augment coarse-scale 3D faces with synthesized fine-scale geometric wrinkles. By formulating the wrinkle generation problem as a supervised generation task, we implicitly model the continuous space of face wrinkles via a compact generative model, such that plausible face wrinkles can be generated through effective sampling and interpolation in the space. We also introduce a complete pipeline to transfer the synthesized wrinkles between faces with different shapes and topologies. Through many experiments, we demonstrate our method can robustly synthesize plausible fine-scale wrinkles on a variety of coarse-scale 3D faces with different shapes and expressions.},
 bibtype = {article},
 author = {Deng, Qixin and Ma, Luming and Jin, Aobo and Bi, Huikun and Le, Binh Huy and Deng, Zhigang},
 doi = {10.1109/TVCG.2021.3051251},
 journal = {IEEE Transactions on Visualization and Computer Graphics},
 number = {01}
}

Realistic 3D facial modeling and animation have been increasingly used in many graphics, animation, and virtual reality applications. However, generating realistic fine-scale wrinkles on 3D faces, in particular, on animated 3D faces, is still a challenging problem that is far away from being resolved. In this paper we propose an end-to-end system to automatically augment coarse-scale 3D faces with synthesized fine-scale geometric wrinkles. By formulating the wrinkle generation problem as a supervised generation task, we implicitly model the continuous space of face wrinkles via a compact generative model, such that plausible face wrinkles can be generated through effective sampling and interpolation in the space. We also introduce a complete pipeline to transfer the synthesized wrinkles between faces with different shapes and topologies. Through many experiments, we demonstrate our method can robustly synthesize plausible fine-scale wrinkles on a variety of coarse-scale 3D faces with different shapes and expressions.

@inproceedings{
 title = {Task-Generic Hierarchical Human Motion Prior using VAEs},
 type = {inproceedings},
 year = {2021},
 keywords = {Animation,Interpolation,Motion estimation,Pose estimation,Predictive models,Three-dimensional displays,Training,n/a},
 pages = {771-781},
 month = {12},
 id = {32e7dd96-4d04-3ba9-8830-59d6d3f34a01},
 created = {2022-03-28T09:45:02.580Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:03:35.776Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {liTaskGenericHierarchicalHuman2021},
 source_type = {inproceedings},
 notes = {ISSN: 2475-7888},
 private_publication = {false},
 abstract = {A deep generative model that describes human motions can benefit a wide range of fundamental computer vision and graphics tasks, such as providing robustness to video-based human pose estimation, predicting complete body movements for motion capture systems during occlusions, and assisting key frame animation with plausible movements. In this paper, we present a method for learning complex human motions independent of specific tasks using a combined global and local latent space to facilitate coarse and fine-grained modeling. Specifically, we propose a hierarchical motion variational autoencoder (HM-VAE) that consists of a 2-level hierarchical latent space. While the global latent space captures the overall global body motion, the local latent space enables to capture the refined poses of the different body parts. We demonstrate the effectiveness of our hierarchical motion variational autoencoder in a variety of tasks including video-based human pose estimation, motion completion from partial observations, and motion synthesis from sparse key-frames. Even though, our model has not been trained for any of these tasks specifically, it provides superior performance than task-specific alternatives. Our general-purpose human motion prior model can fix corrupted human body animations and generate complete movements from incomplete observations.},
 bibtype = {inproceedings},
 author = {Li, Jiaman and Villegas, Ruben and Ceylan, Duygu and Yang, Jimei and Kuang, Zhengfei and Li, Hao and Zhao, Yajie},
 doi = {10.1109/3DV53792.2021.00086},
 booktitle = {2021 International Conference on 3D Vision (3DV)}
}

A deep generative model that describes human motions can benefit a wide range of fundamental computer vision and graphics tasks, such as providing robustness to video-based human pose estimation, predicting complete body movements for motion capture systems during occlusions, and assisting key frame animation with plausible movements. In this paper, we present a method for learning complex human motions independent of specific tasks using a combined global and local latent space to facilitate coarse and fine-grained modeling. Specifically, we propose a hierarchical motion variational autoencoder (HM-VAE) that consists of a 2-level hierarchical latent space. While the global latent space captures the overall global body motion, the local latent space enables to capture the refined poses of the different body parts. We demonstrate the effectiveness of our hierarchical motion variational autoencoder in a variety of tasks including video-based human pose estimation, motion completion from partial observations, and motion synthesis from sparse key-frames. Even though, our model has not been trained for any of these tasks specifically, it provides superior performance than task-specific alternatives. Our general-purpose human motion prior model can fix corrupted human body animations and generate complete movements from incomplete observations.

@article{
 title = {Sparse Data Driven Mesh Deformation},
 type = {article},
 year = {2021},
 keywords = {Computational modeling,Data driven,Deformable models,Geometry,Interpolation,Manifolds,Shape,Strain,large scale deformation,real-time deformation,sparsity},
 pages = {2085-2100},
 volume = {27},
 month = {3},
 id = {0a9c6e1a-47ea-3992-aad8-bf0760490b23},
 created = {2022-03-28T09:45:02.604Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:03:59.342Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {gaoSparseDataDriven2021},
 source_type = {article},
 notes = {Conference Name: IEEE Transactions on Visualization and Computer Graphics},
 private_publication = {false},
 abstract = {Example-based mesh deformation methods are powerful tools for realistic shape editing. However, existing techniques typically combine all the example deformation modes, which can lead to overfitting, i.e., using an overly complicated model to explain the user-specified deformation. This leads to implausible or unstable deformation results, including unexpected global changes outside the region of interest. To address this fundamental limitation, we propose a sparse blending method that automatically selects a smaller number of deformation modes to compactly describe the desired deformation. This along with a suitably chosen deformation basis including spatially localized deformation modes leads to significant advantages, including more meaningful, reliable, and efficient deformations because fewer and localized deformation modes are applied. To cope with large rotations, we develop a simple but effective representation based on polar decomposition of deformation gradients, which resolves the ambiguity of large global rotations using an as-consistent-as-possible global optimization. This simple representation has a closed form solution for derivatives, making it efficient for our sparse localized representation and thus ensuring interactive performance. Experimental results show that our method outperforms state-of-the-art data-driven mesh deformation methods, for both quality of results and efficiency.},
 bibtype = {article},
 author = {Gao, Lin and Lai, Yu-Kun and Yang, Jie and Zhang, Ling-Xiao and Xia, Shihong and Kobbelt, Leif},
 doi = {10.1109/TVCG.2019.2941200},
 journal = {IEEE Transactions on Visualization and Computer Graphics},
 number = {3}
}

Example-based mesh deformation methods are powerful tools for realistic shape editing. However, existing techniques typically combine all the example deformation modes, which can lead to overfitting, i.e., using an overly complicated model to explain the user-specified deformation. This leads to implausible or unstable deformation results, including unexpected global changes outside the region of interest. To address this fundamental limitation, we propose a sparse blending method that automatically selects a smaller number of deformation modes to compactly describe the desired deformation. This along with a suitably chosen deformation basis including spatially localized deformation modes leads to significant advantages, including more meaningful, reliable, and efficient deformations because fewer and localized deformation modes are applied. To cope with large rotations, we develop a simple but effective representation based on polar decomposition of deformation gradients, which resolves the ambiguity of large global rotations using an as-consistent-as-possible global optimization. This simple representation has a closed form solution for derivatives, making it efficient for our sparse localized representation and thus ensuring interactive performance. Experimental results show that our method outperforms state-of-the-art data-driven mesh deformation methods, for both quality of results and efficiency.

@article{
 title = {On relationships between the Pearson and the distance correlation coefficients},
 type = {article},
 year = {2021},
 keywords = {Distance correlation,Distance covariance,Mixture distribution,Pearson correlation,Rademacher distribution},
 pages = {108960},
 volume = {169},
 websites = {https://www.sciencedirect.com/science/article/pii/S0167715220302637},
 month = {2},
 id = {54d6eda3-9dcc-3de9-8891-15fdd39d1fd2},
 created = {2022-03-28T09:45:03.188Z},
 accessed = {2022-03-26},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:04:43.569Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {edelmannRelationshipsPearsonDistance2021},
 source_type = {article},
 private_publication = {false},
 abstract = {In this paper we show that for any fixed Pearson correlation coefficient strictly between −1 and 1, the distance correlation coefficient can take any value in the open unit interval (0,1).},
 bibtype = {article},
 author = {Edelmann, Dominic and Móri, Tamás F and Székely, Gábor J},
 doi = {10.1016/j.spl.2020.108960},
 journal = {Statistics \& Probability Letters}
}

In this paper we show that for any fixed Pearson correlation coefficient strictly between −1 and 1, the distance correlation coefficient can take any value in the open unit interval (0,1).

@article{
 title = {Tutorial on Variational Autoencoders},
 type = {article},
 year = {2021},
 keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
 websites = {http://arxiv.org/abs/1606.05908},
 month = {1},
 id = {ad51d3c4-4e54-346a-8c24-25c3db992d38},
 created = {2022-03-28T09:45:03.303Z},
 accessed = {2022-03-26},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:05:22.545Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {doerschTutorialVariationalAutoencoders2021},
 source_type = {article},
 notes = {arXiv: 1606.05908},
 private_publication = {false},
 abstract = {In just three years, Variational Autoencoders (VAEs) have emerged as one of the most popular approaches to unsupervised learning of complicated distributions. VAEs are appealing because they are built on top of standard function approximators (neural networks), and can be trained with stochastic gradient descent. VAEs have already shown promise in generating many kinds of complicated data, including handwritten digits, faces, house numbers, CIFAR images, physical models of scenes, segmentation, and predicting the future from static images. This tutorial introduces the intuitions behind VAEs, explains the mathematics behind them, and describes some empirical behavior. No prior knowledge of variational Bayesian methods is assumed.},
 bibtype = {article},
 author = {Doersch, Carl},
 journal = {arXiv:1606.05908 [cs, stat]}
}

In just three years, Variational Autoencoders (VAEs) have emerged as one of the most popular approaches to unsupervised learning of complicated distributions. VAEs are appealing because they are built on top of standard function approximators (neural networks), and can be trained with stochastic gradient descent. VAEs have already shown promise in generating many kinds of complicated data, including handwritten digits, faces, house numbers, CIFAR images, physical models of scenes, segmentation, and predicting the future from static images. This tutorial introduces the intuitions behind VAEs, explains the mathematics behind them, and describes some empirical behavior. No prior knowledge of variational Bayesian methods is assumed.

@article{
 title = {Mobil robotkar felhasználása a mezőgazdaságban: Mobile robotic manipulator for precision agriculture},
 type = {article},
 year = {2021},
 keywords = {navigáció},
 pages = {55-61},
 websites = {https://ojs.emt.ro/index.php/enelko-szamokt/article/view/622},
 month = {10},
 id = {437ebb46-16b4-3dd9-991f-caf5bcfb0459},
 created = {2022-03-28T09:45:03.363Z},
 accessed = {2022-01-26},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:04:59.575Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {orsMobilRobotkarFelhasznalasa2021},
 source_type = {article},
 short_title = {Mobil robotkar felhasználása a mezőgazdaságban},
 private_publication = {false},
 abstract = {In this paper a mobile manipulator arm for automated harvesting is proposed. This device can be useful in such places, where the human resources are too expensive. The main parts of the assembly are a four-wheeled robot and a robotic arm, depth sensing camera and embedded GPU device. The device can navigate through obstacles without hitting them, and find a path to the targeted plant. With a camera it can recognize the fruits, and based on visual servoing it can crop them.
Kivonat
Ebben a dolgozatban bemutatunk egy mobil robot kart, amely nagy segítséget nyújthat az agrikultúrában, főleg olyan helyeken, ahol az emberi munkaerőhiány van. Az eszköz főbb alkotórészei egy négykerekű mozgó robot, valamint egy robotkar. A szerkezet képes elnavigálni az akadályok között és amikor talál egy növényt, képes felismerni rajta a gyümölcsöt, majd azt leszedni, és elszállítani.},
 bibtype = {article},
 author = {Örs, KÖLLŐ Magor and Szilárd, Molnár and Levente, Tamás},
 journal = {Energetika-Elektrotechnika – Számítástechnika és Oktatás Multi-konferencia}
}

In this paper a mobile manipulator arm for automated harvesting is proposed. This device can be useful in such places, where the human resources are too expensive. The main parts of the assembly are a four-wheeled robot and a robotic arm, depth sensing camera and embedded GPU device. The device can navigate through obstacles without hitting them, and find a path to the targeted plant. With a camera it can recognize the fruits, and based on visual servoing it can crop them. Kivonat Ebben a dolgozatban bemutatunk egy mobil robot kart, amely nagy segítséget nyújthat az agrikultúrában, főleg olyan helyeken, ahol az emberi munkaerőhiány van. Az eszköz főbb alkotórészei egy négykerekű mozgó robot, valamint egy robotkar. A szerkezet képes elnavigálni az akadályok között és amikor talál egy növényt, képes felismerni rajta a gyümölcsöt, majd azt leszedni, és elszállítani.

@article{
 title = {Learned Point Cloud Geometry Compression},
 type = {article},
 year = {2021},
 keywords = {Computer Science - Computer Vision and Pattern Rec,Electrical Engineering and Systems Science - Imag},
 pages = {4909-4923},
 volume = {31},
 websites = {http://arxiv.org/abs/1909.12037},
 month = {12},
 id = {899c47d0-e122-3173-a877-e973e289ad74},
 created = {2022-03-28T09:45:03.904Z},
 accessed = {2022-01-05},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:06:27.499Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {wangLearnedPointCloud2021},
 source_type = {article},
 notes = {arXiv: 1909.12037},
 private_publication = {false},
 abstract = {This paper presents a novel end-to-end Learned Point Cloud Geometry Compression (a.k.a., Learned-PCGC) framework, to efficiently compress the point cloud geometry (PCG) using deep neural networks (DNN) based variational autoencoders (VAE). In our approach, PCG is first voxelized, scaled and partitioned into non-overlapped 3D cubes, which is then fed into stacked 3D convolutions for compact latent feature and hyperprior generation. Hyperpriors are used to improve the conditional probability modeling of latent features. A weighted binary cross-entropy (WBCE) loss is applied in training while an adaptive thresholding is used in inference to remove unnecessary voxels and reduce the distortion. Objectively, our method exceeds the geometry-based point cloud compression (G-PCC) algorithm standardized by well-known Moving Picture Experts Group (MPEG) with a significant performance margin, e.g., at least 60\% BD-Rate (Bjontegaard Delta Rate) gains, using common test datasets. Subjectively, our method has presented better visual quality with smoother surface reconstruction and appealing details, in comparison to all existing MPEG standard compliant PCC methods. Our method requires about 2.5MB parameters in total, which is a fairly small size for practical implementation, even on embedded platform. Additional ablation studies analyze a variety of aspects (e.g., cube size, kernels, etc) to explore the application potentials of our learned-PCGC.},
 bibtype = {article},
 author = {Wang, Jianqiang and Zhu, Hao and Ma, Zhan and Chen, Tong and Liu, Haojie and Shen, Qiu},
 doi = {10.1109/TCSVT.2021.3051377},
 journal = {IEEE Transactions on Circuits and Systems for Video Technology},
 number = {12}
}

This paper presents a novel end-to-end Learned Point Cloud Geometry Compression (a.k.a., Learned-PCGC) framework, to efficiently compress the point cloud geometry (PCG) using deep neural networks (DNN) based variational autoencoders (VAE). In our approach, PCG is first voxelized, scaled and partitioned into non-overlapped 3D cubes, which is then fed into stacked 3D convolutions for compact latent feature and hyperprior generation. Hyperpriors are used to improve the conditional probability modeling of latent features. A weighted binary cross-entropy (WBCE) loss is applied in training while an adaptive thresholding is used in inference to remove unnecessary voxels and reduce the distortion. Objectively, our method exceeds the geometry-based point cloud compression (G-PCC) algorithm standardized by well-known Moving Picture Experts Group (MPEG) with a significant performance margin, e.g., at least 60\% BD-Rate (Bjontegaard Delta Rate) gains, using common test datasets. Subjectively, our method has presented better visual quality with smoother surface reconstruction and appealing details, in comparison to all existing MPEG standard compliant PCC methods. Our method requires about 2.5MB parameters in total, which is a fairly small size for practical implementation, even on embedded platform. Additional ablation studies analyze a variety of aspects (e.g., cube size, kernels, etc) to explore the application potentials of our learned-PCGC.

@inproceedings{
 title = {Private-Shared Disentangled Multimodal VAE for Learning of Latent Representations},
 type = {inproceedings},
 year = {2021},
 pages = {1692-1700},
 websites = {https://openaccess.thecvf.com/content/CVPR2021W/MULA/html/Lee_Private-Shared_Disentangled_Multimodal_VAE_for_Learning_of_Latent_Representations_CVPRW_2021_paper.html},
 id = {246f0810-870d-3ed4-8765-aef5efe93149},
 created = {2022-03-28T09:45:04.311Z},
 accessed = {2021-09-29},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:07:24.837Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {leePrivateSharedDisentangledMultimodal2021},
 source_type = {inproceedings},
 private_publication = {false},
 bibtype = {inproceedings},
 author = {Lee, Mihee and Pavlovic, Vladimir}
}

@inproceedings{
 title = {NeRF-VAE: A Geometry Aware 3D Scene Generative Model},
 type = {inproceedings},
 year = {2021},
 pages = {5742-5752},
 websites = {https://proceedings.mlr.press/v139/kosiorek21a.html},
 month = {7},
 publisher = {PMLR},
 id = {90ecafa0-c3e0-3a6c-80b9-e5ec026decfe},
 created = {2022-03-28T09:45:04.596Z},
 accessed = {2022-03-27},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:07:34.628Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {kosiorekNeRFVAEGeometryAware2021a},
 source_type = {inproceedings},
 short_title = {NeRF-VAE},
 notes = {ISSN: 2640-3498},
 private_publication = {false},
 abstract = {We propose NeRF-VAE, a 3D scene generative model that incorporates geometric structure via Neural Radiance Fields (NeRF) and differentiable volume rendering. In contrast to NeRF, our model takes into account shared structure across scenes, and is able to infer the structure of a novel scene—without the need to re-train—using amortized inference. NeRF-VAE’s explicit 3D rendering process further contrasts previous generative models with convolution-based rendering which lacks geometric structure. Our model is a VAE that learns a distribution over radiance fields by conditioning them on a latent scene representation. We show that, once trained, NeRF-VAE is able to infer and render geometrically-consistent scenes from previously unseen 3D environments of synthetic scenes using very few input images. We further demonstrate that NeRF-VAE generalizes well to out-of-distribution cameras, while convolutional models do not. Finally, we introduce and study an attention-based conditioning mechanism of NeRF-VAE’s decoder, which improves model performance.},
 bibtype = {inproceedings},
 author = {Kosiorek, Adam R and Strathmann, Heiko and Zoran, Daniel and Moreno, Pol and Schneider, Rosalia and Mokra, Sona and Rezende, Danilo Jimenez},
 booktitle = {Proceedings of the 38th International Conference on Machine Learning}
}

We propose NeRF-VAE, a 3D scene generative model that incorporates geometric structure via Neural Radiance Fields (NeRF) and differentiable volume rendering. In contrast to NeRF, our model takes into account shared structure across scenes, and is able to infer the structure of a novel scene—without the need to re-train—using amortized inference. NeRF-VAE’s explicit 3D rendering process further contrasts previous generative models with convolution-based rendering which lacks geometric structure. Our model is a VAE that learns a distribution over radiance fields by conditioning them on a latent scene representation. We show that, once trained, NeRF-VAE is able to infer and render geometrically-consistent scenes from previously unseen 3D environments of synthetic scenes using very few input images. We further demonstrate that NeRF-VAE generalizes well to out-of-distribution cameras, while convolutional models do not. Finally, we introduce and study an attention-based conditioning mechanism of NeRF-VAE’s decoder, which improves model performance.

@article{
 title = {NVAE: A Deep Hierarchical Variational Autoencoder},
 type = {article},
 year = {2021},
 keywords = {Computer Science - Computer Vision and Pattern Re,Computer Science - Machine Learning,Statistics - Machine Learning},
 websites = {http://arxiv.org/abs/2007.03898},
 month = {1},
 id = {040f5984-81fd-351b-aec7-93f7b60352ed},
 created = {2022-03-28T09:45:04.637Z},
 accessed = {2022-03-22},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:07:42.278Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {vahdatNVAEDeepHierarchical2021},
 source_type = {article},
 short_title = {NVAE},
 notes = {arXiv: 2007.03898},
 private_publication = {false},
 abstract = {Normalizing flows, autoregressive models, variational autoencoders (VAEs), and deep energy-based models are among competing likelihood-based frameworks for deep generative learning. Among them, VAEs have the advantage of fast and tractable sampling and easy-to-access encoding networks. However, they are currently outperformed by other models such as normalizing flows and autoregressive models. While the majority of the research in VAEs is focused on the statistical challenges, we explore the orthogonal direction of carefully designing neural architectures for hierarchical VAEs. We propose Nouveau VAE (NVAE), a deep hierarchical VAE built for image generation using depth-wise separable convolutions and batch normalization. NVAE is equipped with a residual parameterization of Normal distributions and its training is stabilized by spectral regularization. We show that NVAE achieves state-of-the-art results among non-autoregressive likelihood-based models on the MNIST, CIFAR-10, CelebA 64, and CelebA HQ datasets and it provides a strong baseline on FFHQ. For example, on CIFAR-10, NVAE pushes the state-of-the-art from 2.98 to 2.91 bits per dimension, and it produces high-quality images on CelebA HQ. To the best of our knowledge, NVAE is the first successful VAE applied to natural images as large as 256\$\textbackslashtimes\$256 pixels. The source code is available at https://github.com/NVlabs/NVAE .},
 bibtype = {article},
 author = {Vahdat, Arash and Kautz, Jan},
 journal = {arXiv:2007.03898 [cs, stat]}
}

Normalizing flows, autoregressive models, variational autoencoders (VAEs), and deep energy-based models are among competing likelihood-based frameworks for deep generative learning. Among them, VAEs have the advantage of fast and tractable sampling and easy-to-access encoding networks. However, they are currently outperformed by other models such as normalizing flows and autoregressive models. While the majority of the research in VAEs is focused on the statistical challenges, we explore the orthogonal direction of carefully designing neural architectures for hierarchical VAEs. We propose Nouveau VAE (NVAE), a deep hierarchical VAE built for image generation using depth-wise separable convolutions and batch normalization. NVAE is equipped with a residual parameterization of Normal distributions and its training is stabilized by spectral regularization. We show that NVAE achieves state-of-the-art results among non-autoregressive likelihood-based models on the MNIST, CIFAR-10, CelebA 64, and CelebA HQ datasets and it provides a strong baseline on FFHQ. For example, on CIFAR-10, NVAE pushes the state-of-the-art from 2.98 to 2.91 bits per dimension, and it produces high-quality images on CelebA HQ. To the best of our knowledge, NVAE is the first successful VAE applied to natural images as large as 256\$\textbackslashtimes\$256 pixels. The source code is available at https://github.com/NVlabs/NVAE .

@article{
 title = {Very Deep VAEs Generalize Autoregressive Models and Can Outperform Them on Images},
 type = {article},
 year = {2021},
 keywords = {Computer Science - Computer Vision and Pattern Rec,Computer Science - Machine Learning},
 websites = {http://arxiv.org/abs/2011.10650},
 month = {3},
 id = {005bef88-e525-3b32-a0b3-5575ecdba094},
 created = {2022-03-28T09:45:06.105Z},
 accessed = {2022-03-27},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-01T09:16:29.684Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {childVeryDeepVAEs2021},
 source_type = {article},
 notes = {arXiv: 2011.10650},
 private_publication = {false},
 abstract = {We present a hierarchical VAE that, for the first time, generates samples quickly while outperforming the PixelCNN in log-likelihood on all natural image benchmarks. We begin by observing that, in theory, VAEs can actually represent autoregressive models, as well as faster, better models if they exist, when made sufficiently deep. Despite this, autoregressive models have historically outperformed VAEs in log-likelihood. We test if insufficient depth explains why by scaling a VAE to greater stochastic depth than previously explored and evaluating it CIFAR-10, ImageNet, and FFHQ. In comparison to the PixelCNN, these very deep VAEs achieve higher likelihoods, use fewer parameters, generate samples thousands of times faster, and are more easily applied to high-resolution images. Qualitative studies suggest this is because the VAE learns efficient hierarchical visual representations. We release our source code and models at https://github.com/openai/vdvae.},
 bibtype = {article},
 author = {Child, Rewon},
 journal = {arXiv:2011.10650 [cs]}
}

We present a hierarchical VAE that, for the first time, generates samples quickly while outperforming the PixelCNN in log-likelihood on all natural image benchmarks. We begin by observing that, in theory, VAEs can actually represent autoregressive models, as well as faster, better models if they exist, when made sufficiently deep. Despite this, autoregressive models have historically outperformed VAEs in log-likelihood. We test if insufficient depth explains why by scaling a VAE to greater stochastic depth than previously explored and evaluating it CIFAR-10, ImageNet, and FFHQ. In comparison to the PixelCNN, these very deep VAEs achieve higher likelihoods, use fewer parameters, generate samples thousands of times faster, and are more easily applied to high-resolution images. Qualitative studies suggest this is because the VAE learns efficient hierarchical visual representations. We release our source code and models at https://github.com/openai/vdvae.

@inproceedings{
 title = {ReZero is all you need: fast convergence at large depth},
 type = {inproceedings},
 year = {2021},
 pages = {1352-1361},
 websites = {https://proceedings.mlr.press/v161/bachlechner21a.html},
 month = {12},
 publisher = {PMLR},
 id = {d84b7a5f-5fc7-344e-99b2-816e14d1a21f},
 created = {2022-03-28T09:45:06.262Z},
 accessed = {2022-03-27},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-01T09:16:36.292Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {bachlechnerReZeroAllYou2021},
 source_type = {inproceedings},
 short_title = {ReZero is all you need},
 notes = {ISSN: 2640-3498},
 private_publication = {false},
 abstract = {Deep networks often suffer from vanishing or exploding gradients due to inefficient signal propagation, leading to long training times or convergence difficulties. Various architecture designs, sophisticated residual-style networks, and initialization schemes have been shown to improve deep signal propagation. Recently, Pennington et al. [2017] used free probability theory to show that dynamical isometry plays an integral role in efficient deep learning. We show that the simplest architecture change of gating each residual connection using a single zero-initialized parameter satisfies initial dynamical isometry and outperforms more complex approaches. Although much simpler than its predecessors, this gate enables training thousands of fully connected layers with fast convergence and better test performance for ResNets trained on an image recognition task. We apply this technique to language modeling and find that we can easily train 120-layer Transformers. When applied to 12 layer Transformers, it converges 56\% faster.},
 bibtype = {inproceedings},
 author = {Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Henry and Cottrell, Gary and McAuley, Julian},
 booktitle = {Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence}
}

Deep networks often suffer from vanishing or exploding gradients due to inefficient signal propagation, leading to long training times or convergence difficulties. Various architecture designs, sophisticated residual-style networks, and initialization schemes have been shown to improve deep signal propagation. Recently, Pennington et al. [2017] used free probability theory to show that dynamical isometry plays an integral role in efficient deep learning. We show that the simplest architecture change of gating each residual connection using a single zero-initialized parameter satisfies initial dynamical isometry and outperforms more complex approaches. Although much simpler than its predecessors, this gate enables training thousands of fully connected layers with fast convergence and better test performance for ResNets trained on an image recognition task. We apply this technique to language modeling and find that we can easily train 120-layer Transformers. When applied to 12 layer Transformers, it converges 56\% faster.

@inproceedings{
 title = {3D Semantic Label Transfer in Human-Robot Collaboration},
 type = {inproceedings},
 year = {2021},
 pages = {2602-2611},
 websites = {https://openaccess.thecvf.com/content/ICCV2021W/CVinHRC/html/Rozenberszki_3D_Semantic_Label_Transfer_in_Human-Robot_Collaboration_ICCVW_2021_paper.html},
 id = {c4c9ba73-1d7b-30bc-a927-8028aa77f395},
 created = {2022-03-28T09:45:06.498Z},
 accessed = {2021-10-20},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T07:59:40.722Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {rozenberszki3DSemanticLabel2021},
 source_type = {inproceedings},
 private_publication = {false},
 bibtype = {inproceedings},
 author = {Rozenberszki, David and Soros, Gabor and Szeier, Szilvia and Lorincz, Andras}
}

@article{
 title = {Automated building change detection with amodal completion of point clouds},
 type = {article},
 year = {2021},
 keywords = {Building information modeling,Change detection,Deep learning,Point cloud,Point cloud completion},
 pages = {103568},
 volume = {124},
 websites = {https://www.sciencedirect.com/science/article/pii/S0926580521000194},
 month = {4},
 id = {8249925c-4de8-394b-8e2e-f7bea0fd31a3},
 created = {2022-03-28T09:45:06.711Z},
 accessed = {2022-03-22},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-03-29T08:00:45.489Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {czerniawskiAutomatedBuildingChange2021},
 source_type = {article},
 private_publication = {false},
 abstract = {When updating digital models of existing buildings, changes in the built environment are detected by comparing outdated BIMs with captured point clouds representing current conditions. Here we show that point cloud completion (i.e. automated filling-in of missing data) improves the accuracy of change detection. We perform point cloud completion using a hierarchical deep variational autoencoder (a type of artificial neural network) modified to include skip connections between the convolution and deconvolution layers. The resulting receiver operating characteristic curve shows that completion boosts change detection performance from a total area under the curve of 0.55 to 0.75. Completion achieves this by eliminating differences between the BIM and point cloud inputs that are a consequence of incompleteness while distilling the differences due to building change. We anticipate that automated change detection methods with resilience to imperfect data will become more critical as automated building analyses become increasingly abstracted from data collection.},
 bibtype = {article},
 author = {Czerniawski, Thomas and Ma, Jong Won and Fernanda Leite, undefined},
 doi = {10.1016/j.autcon.2021.103568},
 journal = {Automation in Construction}
}

When updating digital models of existing buildings, changes in the built environment are detected by comparing outdated BIMs with captured point clouds representing current conditions. Here we show that point cloud completion (i.e. automated filling-in of missing data) improves the accuracy of change detection. We perform point cloud completion using a hierarchical deep variational autoencoder (a type of artificial neural network) modified to include skip connections between the convolution and deconvolution layers. The resulting receiver operating characteristic curve shows that completion boosts change detection performance from a total area under the curve of 0.55 to 0.75. Completion achieves this by eliminating differences between the BIM and point cloud inputs that are a consequence of incompleteness while distilling the differences due to building change. We anticipate that automated change detection methods with resilience to imperfect data will become more critical as automated building analyses become increasingly abstracted from data collection.

@article{
 title = {Strategies of attack–defense game for wireless sensor networks considering the effect of confidence level in fuzzy environment},
 type = {article},
 year = {2021},
 keywords = {Fuzzy set,Malware,Reliability analysis approach,Wireless sensor network},
 pages = {104238},
 volume = {102},
 websites = {https://doi.org/10.1016/j.engappai.2021.104238},
 publisher = {Elsevier Ltd},
 id = {8f099e30-11fb-307b-b709-35d356e5f8b3},
 created = {2022-04-05T05:35:07.815Z},
 file_attached = {false},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:07.815Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {It is a common case that Wireless Sensor Networks are attacked by malware in the real world. According to the game theory, the action of attack–defense between Wireless Sensor Network(WSN) and malware can be regarded as a game. While substantial efforts have been made to address this issue, most of these efforts have predominantly focused on the analysis of attack–defense game in the known environment. Given that the process of gaming in real world often contains a lot of fuzzy information, we extend the focus in this line by considering the fuzzy exterior environment. Specifically, we assume the WSN attack–defense Stackelberg game is in the fuzzy environment by using fuzzy variable. Then Stackelberg game theory is utilized to calculate the equilibrium solutions of the introduced maximax chance-constrained model and minimax chance-constrained model. Based on the simulation data, this study demonstrates the confidence levels and decision perspectives affect the optimal strategy of WSN and the reliability of WSN. Finally, the novel analytical method is compared with the non-fuzzy WSN attack–defense game method. The analysis shows that the novel approach is optimal in terms of predicting the behavior of malware in resisting the attack of malware.},
 bibtype = {article},
 author = {Wu, Yingfu and Kang, Bingyi and Wu, Hao},
 doi = {10.1016/j.engappai.2021.104238},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {April}
}

It is a common case that Wireless Sensor Networks are attacked by malware in the real world. According to the game theory, the action of attack–defense between Wireless Sensor Network(WSN) and malware can be regarded as a game. While substantial efforts have been made to address this issue, most of these efforts have predominantly focused on the analysis of attack–defense game in the known environment. Given that the process of gaming in real world often contains a lot of fuzzy information, we extend the focus in this line by considering the fuzzy exterior environment. Specifically, we assume the WSN attack–defense Stackelberg game is in the fuzzy environment by using fuzzy variable. Then Stackelberg game theory is utilized to calculate the equilibrium solutions of the introduced maximax chance-constrained model and minimax chance-constrained model. Based on the simulation data, this study demonstrates the confidence levels and decision perspectives affect the optimal strategy of WSN and the reliability of WSN. Finally, the novel analytical method is compared with the non-fuzzy WSN attack–defense game method. The analysis shows that the novel approach is optimal in terms of predicting the behavior of malware in resisting the attack of malware.

@article{
 title = {Regression-based methods for face alignment: A survey},
 type = {article},
 year = {2021},
 keywords = {Face alignment,Facial feature localization,Facial landmarks detection,Regression,Survey},
 volume = {178},
 id = {c6757daa-bd68-3d66-a352-21e01b14921f},
 created = {2022-04-05T05:35:07.934Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-07T06:10:54.749Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {Face alignment is the process of determining a face shape given its location and size in an image. It is used as a basis for other facial analysis tasks and for human-machine interaction and augmented reality applications. It is a challenging problem due to the extremely high variability in facial appearance affected by many external (illumination, occlusion, head pose) and internal factors (race, facial expression). However, advances in deep learning combined with domain-related knowledge from previous research recently demonstrated impressive results nearly saturating the unconstrained benchmark data sets. The focus is shifting towards reducing the computational burden of the face alignment models since real-time performance is required for such a highly dynamic task. Furthermore, many applications target devices on the edge with limited computational power which puts even greater emphasis on computational efficiency. We present the latest development in regression-based approaches that have led towards nearly solving the face alignment problem in an unconstrained scenario. Various regression architectures are systematically explored and recent training techniques discussed in the context of face alignment. Finally, a benchmark comparison of the most successful methods is presented, taking into account execution time as well, to provide a comprehensive overview of this dynamic research field.},
 bibtype = {article},
 author = {Gogić, Ivan and Ahlberg, Jörgen and Pandžić, Igor S.},
 doi = {10.1016/j.sigpro.2020.107755},
 journal = {Signal Processing}
}

Face alignment is the process of determining a face shape given its location and size in an image. It is used as a basis for other facial analysis tasks and for human-machine interaction and augmented reality applications. It is a challenging problem due to the extremely high variability in facial appearance affected by many external (illumination, occlusion, head pose) and internal factors (race, facial expression). However, advances in deep learning combined with domain-related knowledge from previous research recently demonstrated impressive results nearly saturating the unconstrained benchmark data sets. The focus is shifting towards reducing the computational burden of the face alignment models since real-time performance is required for such a highly dynamic task. Furthermore, many applications target devices on the edge with limited computational power which puts even greater emphasis on computational efficiency. We present the latest development in regression-based approaches that have led towards nearly solving the face alignment problem in an unconstrained scenario. Various regression architectures are systematically explored and recent training techniques discussed in the context of face alignment. Finally, a benchmark comparison of the most successful methods is presented, taking into account execution time as well, to provide a comprehensive overview of this dynamic research field.

@article{
 title = {A comprehensive survey on 2D multi-person pose estimation methods},
 type = {article},
 year = {2021},
 keywords = {Deep learning,Multi-person pose estimation,Survey},
 pages = {104260},
 volume = {102},
 websites = {https://doi.org/10.1016/j.engappai.2021.104260},
 publisher = {Elsevier Ltd},
 id = {47ff27f2-6237-309e-a719-e22b4c33ad92},
 created = {2022-04-05T05:35:08.081Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-07T06:10:54.786Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {Human pose estimation is a fundamental yet challenging computer vision task and studied by many researchers around the world in recent years. As a basic task in computer vision, multi-person pose estimation is the core component for many practical applications. This paper extensively reviews recent works on multi-person pose estimation. Specifically, we illustrate and analyze popular methods in detail and compare their pros and cons to fill in the gaps existing in other surveys. In addition, the commonly used datasets, evaluation metrics, and open-source systems are also introduced respectively. Finally, we summarize the development of multi-person pose estimation frameworks and discuss the research trends.},
 bibtype = {article},
 author = {Wang, Chen and Zhang, Feng and Ge, Shuzhi Sam},
 doi = {10.1016/j.engappai.2021.104260},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {April}
}

Human pose estimation is a fundamental yet challenging computer vision task and studied by many researchers around the world in recent years. As a basic task in computer vision, multi-person pose estimation is the core component for many practical applications. This paper extensively reviews recent works on multi-person pose estimation. Specifically, we illustrate and analyze popular methods in detail and compare their pros and cons to fill in the gaps existing in other surveys. In addition, the commonly used datasets, evaluation metrics, and open-source systems are also introduced respectively. Finally, we summarize the development of multi-person pose estimation frameworks and discuss the research trends.

@article{
 title = {Variational graph autoencoders for multiview canonical correlation analysis},
 type = {article},
 year = {2021},
 keywords = {Canonical correlation analysis,Dimensionality reduction,Graph neurals networks,Multiview representation learning,Variational inference},
 pages = {108182},
 volume = {188},
 websites = {https://doi.org/10.1016/j.sigpro.2021.108182},
 publisher = {Elsevier B.V.},
 id = {3608c6aa-c36f-3a4b-9bbd-3eb90edb125b},
 created = {2022-04-05T05:35:08.236Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-05T05:35:14.172Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {We present a novel approach for multiview canonical correlation analysis based on a variational graph neural network model. We propose a nonlinear model which takes into account the available graph-based geometric constraints while being scalable to large-scale datasets with multiple views. This model combines the probabilistic interpretation of CCA with an autoencoder architecture based on graph convolutional neural network layers. Experiments with the proposed method are conducted on classification, clustering, and recommendation tasks on real datasets. The algorithm is competitive with state-of-the-art multiview representation learning techniques, in addition to being scalable and robust to instances with missing views.},
 bibtype = {article},
 author = {Kaloga, Yacouba and Borgnat, Pierre and Chepuri, Sundeep Prabhakar and Abry, Patrice and Habrard, Amaury},
 doi = {10.1016/j.sigpro.2021.108182},
 journal = {Signal Processing}
}

We present a novel approach for multiview canonical correlation analysis based on a variational graph neural network model. We propose a nonlinear model which takes into account the available graph-based geometric constraints while being scalable to large-scale datasets with multiple views. This model combines the probabilistic interpretation of CCA with an autoencoder architecture based on graph convolutional neural network layers. Experiments with the proposed method are conducted on classification, clustering, and recommendation tasks on real datasets. The algorithm is competitive with state-of-the-art multiview representation learning techniques, in addition to being scalable and robust to instances with missing views.

@article{
 title = {A comprehensive survey on digital video forensics: Taxonomy, challenges, and future directions},
 type = {article},
 year = {2021},
 keywords = {Anti-forensics,Computer vision (CV),Deep learning (DL),Digital forensics,Evidence extraction,Forgery detection,Legal aspects,Machine learning (ML),Video forensics,Video forgery},
 pages = {104456},
 volume = {106},
 websites = {https://doi.org/10.1016/j.engappai.2021.104456},
 publisher = {Elsevier Ltd},
 id = {3f503864-0e27-3a81-a5c3-21a385805a70},
 created = {2022-04-07T06:10:54.442Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-04-07T06:10:59.507Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {d44c1c58-0149-4360-9eaf-8e2a9b657b50},
 private_publication = {false},
 abstract = {With the explosive advancements in smartphone technology, video uploading/downloading has become a routine part of digital social networking. Video contents contain valuable information as more incidents are being recorded now than ever before. In this paper, we present a comprehensive survey on information extraction from video contents and forgery detection. In this context, we review various modern techniques such as computer vision and different machine learning (ML) algorithms including deep learning (DL) proposed for video forgery detection. Furthermore, we discuss the persistent general, resource, legal, and technical challenges, as well as challenges in using DL for the problem at hand, such as the theory behind DL, CV, limited datasets, real-time processing, and the challenges with the emergence of ML techniques used with the Internet of Things (IoT)-based heterogeneous devices. Moreover, this survey presents prominent video analysis products used for video forensics investigation and analysis. In summary, this survey provides a detailed and broader investigation about information extraction and forgery detection in video contents under one umbrella, which was not presented yet to the best of our knowledge.},
 bibtype = {article},
 author = {Javed, Abdul Rehman and Jalil, Zunera and Zehra, Wisha and Gadekallu, Thippa Reddy and Suh, Doug Young and Piran, Md Jalil},
 doi = {10.1016/j.engappai.2021.104456},
 journal = {Engineering Applications of Artificial Intelligence},
 number = {August}
}

With the explosive advancements in smartphone technology, video uploading/downloading has become a routine part of digital social networking. Video contents contain valuable information as more incidents are being recorded now than ever before. In this paper, we present a comprehensive survey on information extraction from video contents and forgery detection. In this context, we review various modern techniques such as computer vision and different machine learning (ML) algorithms including deep learning (DL) proposed for video forgery detection. Furthermore, we discuss the persistent general, resource, legal, and technical challenges, as well as challenges in using DL for the problem at hand, such as the theory behind DL, CV, limited datasets, real-time processing, and the challenges with the emergence of ML techniques used with the Internet of Things (IoT)-based heterogeneous devices. Moreover, this survey presents prominent video analysis products used for video forensics investigation and analysis. In summary, this survey provides a detailed and broader investigation about information extraction and forgery detection in video contents under one umbrella, which was not presented yet to the best of our knowledge.

@article{
 title = {Kimera: from SLAM to Spatial Perception with 3D Dynamic Scene Graphs},
 type = {article},
 year = {2021},
 keywords = {Localization,computer vision,mapping,sensing and perception,slam},
 pages = {1510-1546},
 volume = {40},
 websites = {https://arxiv.org/abs/2101.06894v3},
 month = {1},
 publisher = {SAGE Publications Inc.},
 day = {18},
 id = {27141204-b6f1-367d-88b1-95617a96caa5},
 created = {2022-06-06T05:53:43.828Z},
 accessed = {2022-06-06},
 file_attached = {false},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-06-06T05:54:04.302Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {6075c81a-cded-4bc7-822e-6d5f3181ca0d},
 private_publication = {false},
 abstract = {Humans are able to form a complex mental model of the environment they move
in. This mental model captures geometric and semantic aspects of the scene,
describes the environment at multiple levels of abstractions (e.g., objects,
rooms, buildings), includes static and dynamic entities and their relations
(e.g., a person is in a room at a given time). In contrast, current robots'
internal representations still provide a partial and fragmented understanding
of the environment, either in the form of a sparse or dense set of geometric
primitives (e.g., points, lines, planes, voxels) or as a collection of objects.
This paper attempts to reduce the gap between robot and human perception by
introducing a novel representation, a 3D Dynamic Scene Graph(DSG), that
seamlessly captures metric and semantic aspects of a dynamic environment. A DSG
is a layered graph where nodes represent spatial concepts at different levels
of abstraction, and edges represent spatio-temporal relations among nodes. Our
second contribution is Kimera, the first fully automatic method to build a DSG
from visual-inertial data. Kimera includes state-of-the-art techniques for
visual-inertial SLAM, metric-semantic 3D reconstruction, object localization,
human pose and shape estimation, and scene parsing. Our third contribution is a
comprehensive evaluation of Kimera in real-life datasets and photo-realistic
simulations, including a newly released dataset, uHumans2, which simulates a
collection of crowded indoor and outdoor scenes. Our evaluation shows that
Kimera achieves state-of-the-art performance in visual-inertial SLAM, estimates
an accurate 3D metric-semantic mesh model in real-time, and builds a DSG of a
complex indoor environment with tens of objects and humans in minutes. Our
final contribution shows how to use a DSG for real-time hierarchical semantic
path-planning. The core modules in Kimera are open-source.},
 bibtype = {article},
 author = {Rosinol, Antoni and Violette, Andrew and Abate, Marcus and Hughes, Nathan and Chang, Yun and Shi, Jingnan and Gupta, Arjun and Carlone, Luca},
 doi = {10.48550/arxiv.2101.06894},
 journal = {International Journal of Robotics Research},
 number = {12-14}
}

Humans are able to form a complex mental model of the environment they move in. This mental model captures geometric and semantic aspects of the scene, describes the environment at multiple levels of abstractions (e.g., objects, rooms, buildings), includes static and dynamic entities and their relations (e.g., a person is in a room at a given time). In contrast, current robots' internal representations still provide a partial and fragmented understanding of the environment, either in the form of a sparse or dense set of geometric primitives (e.g., points, lines, planes, voxels) or as a collection of objects. This paper attempts to reduce the gap between robot and human perception by introducing a novel representation, a 3D Dynamic Scene Graph(DSG), that seamlessly captures metric and semantic aspects of a dynamic environment. A DSG is a layered graph where nodes represent spatial concepts at different levels of abstraction, and edges represent spatio-temporal relations among nodes. Our second contribution is Kimera, the first fully automatic method to build a DSG from visual-inertial data. Kimera includes state-of-the-art techniques for visual-inertial SLAM, metric-semantic 3D reconstruction, object localization, human pose and shape estimation, and scene parsing. Our third contribution is a comprehensive evaluation of Kimera in real-life datasets and photo-realistic simulations, including a newly released dataset, uHumans2, which simulates a collection of crowded indoor and outdoor scenes. Our evaluation shows that Kimera achieves state-of-the-art performance in visual-inertial SLAM, estimates an accurate 3D metric-semantic mesh model in real-time, and builds a DSG of a complex indoor environment with tens of objects and humans in minutes. Our final contribution shows how to use a DSG for real-time hierarchical semantic path-planning. The core modules in Kimera are open-source.

@article{
 title = {SGMNet: Learning Rotation-Invariant Point Cloud Representations via Sorted Gram Matrix},
 type = {article},
 year = {2021},
 pages = {10448-10457},
 id = {ced97e6c-57e9-342f-92c9-bda70b7a0af9},
 created = {2022-06-21T09:01:27.598Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-06-22T07:14:23.216Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {5177186d-8a3a-482d-a36a-536b8090101c},
 private_publication = {false},
 abstract = {Recently, various works that attempted to introduce rotation invariance to point cloud analysis have devised point-pair features, such as angles and distances. In these methods, however, the point-pair is only comprised of the center point and its adjacent points in a vicinity, which may bring information loss to the local feature representation. In this paper, we instead connect each point densely with all other points in a local neighborhood to compose the point-pairs. Specifically, we present a simple but effective local feature representation, called sorted Gram matrix(SGM), which is not only invariant to arbitrary rotations, but also models the pair-wise relationship of all the points in a neighborhood. In more detail, we utilize vector inner product to model distance- and angle-information between two points, and in a local patch it naturally forms a Gram matrix. In order to guarantee permutation invariance, we sort the correlation value in Gram matrix for each point, therefore this geometric feature names sorted Gram matrix. Furthermore, we mathematically prove that the Gram matrix is rotation-invariant and sufficient to model the inherent structure of a point cloud patch. We then use SGM as features in convolution, which can be readily integrated as a drop-in module into any point-based networks. Finally, we evaluated the proposed method on two widely used datasets, and it outperforms previous state-of-the-arts on both shape classification and part segmentation tasks by a large margin.},
 bibtype = {article},
 author = {Xu, Jianyun and Tang, Xin and Zhu, Yushi and Sun, Jie and Pu, Shiliang},
 doi = {10.1109/ICCV48922.2021.01030},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

Recently, various works that attempted to introduce rotation invariance to point cloud analysis have devised point-pair features, such as angles and distances. In these methods, however, the point-pair is only comprised of the center point and its adjacent points in a vicinity, which may bring information loss to the local feature representation. In this paper, we instead connect each point densely with all other points in a local neighborhood to compose the point-pairs. Specifically, we present a simple but effective local feature representation, called sorted Gram matrix(SGM), which is not only invariant to arbitrary rotations, but also models the pair-wise relationship of all the points in a neighborhood. In more detail, we utilize vector inner product to model distance- and angle-information between two points, and in a local patch it naturally forms a Gram matrix. In order to guarantee permutation invariance, we sort the correlation value in Gram matrix for each point, therefore this geometric feature names sorted Gram matrix. Furthermore, we mathematically prove that the Gram matrix is rotation-invariant and sufficient to model the inherent structure of a point cloud patch. We then use SGM as features in convolution, which can be readily integrated as a drop-in module into any point-based networks. Finally, we evaluated the proposed method on two widely used datasets, and it outperforms previous state-of-the-arts on both shape classification and part segmentation tasks by a large margin.

@article{
 title = {Survey and Evaluation of RGB-D SLAM},
 type = {article},
 year = {2021},
 keywords = {Computer vision,RGB-D SLAM,evaluation,robotics,survey},
 pages = {21367-21387},
 volume = {9},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {e4479ef7-eda8-31b5-9637-d73c81949d48},
 created = {2022-07-05T15:07:35.924Z},
 accessed = {2022-07-05},
 file_attached = {true},
 profile_id = {48fc0258-023d-3602-860e-824092d62c56},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-26T06:52:36.096Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {ed605795-f522-465c-a0b8-5f8a05f7fd5f},
 private_publication = {false},
 abstract = {The traditional visual SLAM systems take the monocular or stereo camera as input sensor, with complex map initialization and map point triangulation steps needed for 3D map reconstruction, which are easy to fail, computationally complex and can cause noisy measurements. The emergence of RGB-D camera which provides RGB image together with depth information breaks this situation. While a number of RGB-D SLAM systems have been proposed in recent years, the current classification research on RGB-D SLAM is very lacking, and their advantages and shortcomings remain unclear regarding different applications and perturbations, such as illumination transformation, noise and rolling shutter effect of sensors. In this paper, we mainly introduced the basic concept and structure of the RGB-D SLAM system, and then introduced the differences between the various RGB-D SLAM systems in the three aspects of tracking, mapping, and loop detection, and we make a classification study on different RGB-D SLAM algorithms according to the three aspect. Furthermore, we discuss some advanced topics and open problems of RGB-D SLAM, hoping that it will help for future exploring. In the end, we conducted a large number of evaluation experiments on multiple RGB-D SLAM systems, and analyzed their advantages and disadvantages, as well as performance differences in different application scenarios, and provided references for researchers and developers.},
 bibtype = {article},
 author = {Zhang, Shishun and Zheng, Longyu and Tao, Wenbing},
 doi = {10.1109/ACCESS.2021.3053188},
 journal = {IEEE Access}
}

The traditional visual SLAM systems take the monocular or stereo camera as input sensor, with complex map initialization and map point triangulation steps needed for 3D map reconstruction, which are easy to fail, computationally complex and can cause noisy measurements. The emergence of RGB-D camera which provides RGB image together with depth information breaks this situation. While a number of RGB-D SLAM systems have been proposed in recent years, the current classification research on RGB-D SLAM is very lacking, and their advantages and shortcomings remain unclear regarding different applications and perturbations, such as illumination transformation, noise and rolling shutter effect of sensors. In this paper, we mainly introduced the basic concept and structure of the RGB-D SLAM system, and then introduced the differences between the various RGB-D SLAM systems in the three aspects of tracking, mapping, and loop detection, and we make a classification study on different RGB-D SLAM algorithms according to the three aspect. Furthermore, we discuss some advanced topics and open problems of RGB-D SLAM, hoping that it will help for future exploring. In the end, we conducted a large number of evaluation experiments on multiple RGB-D SLAM systems, and analyzed their advantages and disadvantages, as well as performance differences in different application scenarios, and provided references for researchers and developers.

@article{
 title = {A Closer Look at Rotation-invariant Deep Point Cloud Analysis},
 type = {article},
 year = {2021},
 pages = {16198-16207},
 id = {354d05d3-a31e-3678-a228-0f7f6db3d834},
 created = {2022-07-28T12:39:24.656Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-07-28T12:39:35.305Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {353ce2e2-5e70-48e5-951f-78dc31fa40d2},
 private_publication = {false},
 abstract = {We consider the deep point cloud analysis tasks where the inputs of the networks are randomly rotated. Recent progress in rotation-invariant point cloud analysis is mainly driven by converting point clouds into their respective canonical poses, and principal component analysis (PCA) is a practical tool to achieve this. Due to the imperfect alignment of PCA, most of the current works are devoted to developing powerful network structures and features to overcome this deficiency, without thoroughly analyzing the PCA-based canonical poses themselves. In this work, we present a detailed study w.r.t. the PCA-based canonical poses of point clouds. Our investigation reveals that the ambiguity problem associated with the PCA-based canonical poses is handled insufficiently in some recent works. To this end, we develop a simple pose selector module for disambiguation, which presents noticeable enhancement (i.e., 5.3% classification accuracy) over state-of-the-art approaches on the challenging real-world dataset.},
 bibtype = {article},
 author = {Li, Feiran and Fujiwara, Kent and Okura, Fumio and Matsushita, Yasuyuki},
 doi = {10.1109/ICCV48922.2021.01591},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

We consider the deep point cloud analysis tasks where the inputs of the networks are randomly rotated. Recent progress in rotation-invariant point cloud analysis is mainly driven by converting point clouds into their respective canonical poses, and principal component analysis (PCA) is a practical tool to achieve this. Due to the imperfect alignment of PCA, most of the current works are devoted to developing powerful network structures and features to overcome this deficiency, without thoroughly analyzing the PCA-based canonical poses themselves. In this work, we present a detailed study w.r.t. the PCA-based canonical poses of point clouds. Our investigation reveals that the ambiguity problem associated with the PCA-based canonical poses is handled insufficiently in some recent works. To this end, we develop a simple pose selector module for disambiguation, which presents noticeable enhancement (i.e., 5.3% classification accuracy) over state-of-the-art approaches on the challenging real-world dataset.

@article{
 title = {On Automatic Data Augmentation for 3D Point Cloud Classification},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2112.06029},
 id = {0ae43773-f5dd-3dfb-905d-66f2ffd3435c},
 created = {2022-08-18T10:51:28.256Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:49.148Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Data augmentation is an important technique to reduce overfitting and improve learning performance, but existing works on data augmentation for 3D point cloud data are based on heuristics. In this work, we instead propose to automatically learn a data augmentation strategy using bilevel optimization. An augmentor is designed in a similar fashion to a conditional generator and is optimized by minimizing a base model's loss on a validation set when the augmented input is used for training the model. This formulation provides a more principled way to learn data augmentation on 3D point clouds. We evaluate our approach on standard point cloud classification tasks and a more challenging setting with pose misalignment between training and validation/test sets. The proposed strategy achieves competitive performance on both tasks and we provide further insight into the augmentor's ability to learn the validation set distribution.},
 bibtype = {article},
 author = {Zhang, Wanyue and Xu, Xun and Liu, Fayao and Zhang, Le and Foo, Chuan-Sheng}
}

Data augmentation is an important technique to reduce overfitting and improve learning performance, but existing works on data augmentation for 3D point cloud data are based on heuristics. In this work, we instead propose to automatically learn a data augmentation strategy using bilevel optimization. An augmentor is designed in a similar fashion to a conditional generator and is optimized by minimizing a base model's loss on a validation set when the augmented input is used for training the model. This formulation provides a more principled way to learn data augmentation on 3D point clouds. We evaluate our approach on standard point cloud classification tasks and a more challenging setting with pose misalignment between training and validation/test sets. The proposed strategy achieves competitive performance on both tasks and we provide further insight into the augmentor's ability to learn the validation set distribution.

@article{
 title = {DeepUME: Learning the Universal Manifold Embedding for Robust Point Cloud Registration},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2112.09938},
 id = {13038110-a185-3bb5-87c7-09d40681e434},
 created = {2022-08-18T10:51:28.260Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:49.139Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Registration of point clouds related by rigid transformations is one of the fundamental problems in computer vision. However, a solution to the practical scenario of aligning sparsely and differently sampled observations in the presence of noise is still lacking. We approach registration in this scenario with a fusion of the closed-form Universal Mani-fold Embedding (UME) method and a deep neural network. The two are combined into a single unified framework, named DeepUME, trained end-to-end and in an unsupervised manner. To successfully provide a global solution in the presence of large transformations, we employ an SO(3)-invariant coordinate system to learn both a joint-resampling strategy of the point clouds and SO(3)-invariant features. These features are then utilized by the geometric UME method for transformation estimation. The parameters of DeepUME are optimized using a metric designed to overcome an ambiguity problem emerging in the registration of symmetric shapes, when noisy scenarios are considered. We show that our hybrid method outperforms state-of-the-art registration methods in various scenarios, and generalizes well to unseen data sets. Our code is publicly available.},
 bibtype = {article},
 author = {Lang, Natalie and Francos, Joseph M.}
}

Registration of point clouds related by rigid transformations is one of the fundamental problems in computer vision. However, a solution to the practical scenario of aligning sparsely and differently sampled observations in the presence of noise is still lacking. We approach registration in this scenario with a fusion of the closed-form Universal Mani-fold Embedding (UME) method and a deep neural network. The two are combined into a single unified framework, named DeepUME, trained end-to-end and in an unsupervised manner. To successfully provide a global solution in the presence of large transformations, we employ an SO(3)-invariant coordinate system to learn both a joint-resampling strategy of the point clouds and SO(3)-invariant features. These features are then utilized by the geometric UME method for transformation estimation. The parameters of DeepUME are optimized using a metric designed to overcome an ambiguity problem emerging in the registration of symmetric shapes, when noisy scenarios are considered. We show that our hybrid method outperforms state-of-the-art registration methods in various scenarios, and generalizes well to unseen data sets. Our code is publicly available.

@article{
 title = {Two Heads are Better than One: Geometric-Latent Attention for Point Cloud Classification and Segmentation},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.00231},
 id = {23a9e521-7d54-31b7-bc0a-31ba054832da},
 created = {2022-08-18T10:51:28.268Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:49.134Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {We present an innovative two-headed attention layer that combines geometric and latent features to segment a 3D scene into semantically meaningful subsets. Each head combines local and global information, using either the geometric or latent features, of a neighborhood of points and uses this information to learn better local relationships. This Geometric-Latent attention layer (Ge-Latto) is combined with a sub-sampling strategy to capture global features. Our method is invariant to permutation thanks to the use of shared-MLP layers, and it can also be used with point clouds with varying densities because the local attention layer does not depend on the neighbor order. Our proposal is simple yet robust, which allows it to achieve competitive results in the ShapeNetPart and ModelNet40 datasets, and the state-of-the-art when segmenting the complex dataset S3DIS, with 69.2% IoU on Area 5, and 89.7% overall accuracy using K-fold cross-validation on the 6 areas.},
 bibtype = {article},
 author = {Cuevas-Velasquez, Hanz and Gallego, Antonio Javier and Fisher, Robert B.}
}

We present an innovative two-headed attention layer that combines geometric and latent features to segment a 3D scene into semantically meaningful subsets. Each head combines local and global information, using either the geometric or latent features, of a neighborhood of points and uses this information to learn better local relationships. This Geometric-Latent attention layer (Ge-Latto) is combined with a sub-sampling strategy to capture global features. Our method is invariant to permutation thanks to the use of shared-MLP layers, and it can also be used with point clouds with varying densities because the local attention layer does not depend on the neighbor order. Our proposal is simple yet robust, which allows it to achieve competitive results in the ShapeNetPart and ModelNet40 datasets, and the state-of-the-art when segmenting the complex dataset S3DIS, with 69.2% IoU on Area 5, and 89.7% overall accuracy using K-fold cross-validation on the 6 areas.

@article{
 title = {Multi-view 3D Reconstruction with Transformers},
 type = {article},
 year = {2021},
 pages = {5702-5711},
 id = {b6154851-8cc1-3fb2-95f7-3fe9420298a3},
 created = {2022-08-18T10:51:28.274Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:09.339Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 abstract = {Deep CNN-based methods have so far achieved the state of the art results in multi-view 3D object reconstruction. Despite the considerable progress, the two core modules of these methods - view feature extraction and multi-view fusion, are usually investigated separately, and the relations among multiple input views are rarely explored. Inspired by the recent great success in Transformer models, we reformulate the multi-view 3D reconstruction as a sequence-to-sequence prediction problem and propose a framework named 3D Volume Transformer. Unlike previous CNN-based methods using a separate design, we unify the feature extraction and view fusion in a single Transformer network. A natural advantage of our design lies in the exploration of view-to-view relationships using self-attention among multiple unordered inputs. On ShapeNet - a large-scale 3D reconstruction benchmark, our method achieves a new state-of-the-art accuracy in multi-view reconstruction with fewer parameters (70% less) than CNN-based methods. Experimental results also suggest the strong scaling capability of our method. Our code will be made publicly available.},
 bibtype = {article},
 author = {Wang, Dan and Cui, Xinrui and Chen, Xun and Zou, Zhengxia and Shi, Tianyang and Salcudean, Septimiu and Wang, Z. Jane and Ward, Rabab},
 doi = {10.1109/ICCV48922.2021.00567},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

Deep CNN-based methods have so far achieved the state of the art results in multi-view 3D object reconstruction. Despite the considerable progress, the two core modules of these methods - view feature extraction and multi-view fusion, are usually investigated separately, and the relations among multiple input views are rarely explored. Inspired by the recent great success in Transformer models, we reformulate the multi-view 3D reconstruction as a sequence-to-sequence prediction problem and propose a framework named 3D Volume Transformer. Unlike previous CNN-based methods using a separate design, we unify the feature extraction and view fusion in a single Transformer network. A natural advantage of our design lies in the exploration of view-to-view relationships using self-attention among multiple unordered inputs. On ShapeNet - a large-scale 3D reconstruction benchmark, our method achieves a new state-of-the-art accuracy in multi-view reconstruction with fewer parameters (70% less) than CNN-based methods. Experimental results also suggest the strong scaling capability of our method. Our code will be made publicly available.

@article{
 title = {Fooling LiDAR Perception via Adversarial Trajectory Perturbation},
 type = {article},
 year = {2021},
 pages = {7878-7887},
 id = {e2ee21cd-6241-3d90-959f-a411e40a3a74},
 created = {2022-08-18T10:51:28.385Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:07.199Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {13ed0c6c-c963-4099-91df-77a297ea5770,db36ed60-3b58-424a-b9a4-a9c7322975f3},
 private_publication = {false},
 abstract = {LiDAR point clouds collected from a moving vehicle are functions of its trajectories, because the sensor motion needs to be compensated to avoid distortions. When autonomous vehicles are sending LiDAR point clouds to deep networks for perception and planning, could the motion compensation consequently become a wide-open backdoor in those networks, due to both the adversarial vulnerability of deep learning and GPS-based vehicle trajectory estimation that is susceptible to wireless spoofing? We demonstrate such possibilities for the first time: instead of directly attacking point cloud coordinates which requires tampering with the raw LiDAR readings, only adversarial spoofing of a self-driving car's trajectory with small perturbations is enough to make safety-critical objects undetectable or detected with incorrect positions. Moreover, polynomial trajectory perturbation is developed to achieve a temporally-smooth and highly-imperceptible attack. Extensive experiments on 3D object detection have shown that such attacks not only lower the performance of the state-of-the-art detectors effectively, but also transfer to other detectors, raising a red flag for the community. The code is available on https://ai4ce.github.io/FLAT/.},
 bibtype = {article},
 author = {Li, Yiming and Wen, Congcong and Juefei-Xu, Felix and Feng, Chen},
 doi = {10.1109/ICCV48922.2021.00780},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

LiDAR point clouds collected from a moving vehicle are functions of its trajectories, because the sensor motion needs to be compensated to avoid distortions. When autonomous vehicles are sending LiDAR point clouds to deep networks for perception and planning, could the motion compensation consequently become a wide-open backdoor in those networks, due to both the adversarial vulnerability of deep learning and GPS-based vehicle trajectory estimation that is susceptible to wireless spoofing? We demonstrate such possibilities for the first time: instead of directly attacking point cloud coordinates which requires tampering with the raw LiDAR readings, only adversarial spoofing of a self-driving car's trajectory with small perturbations is enough to make safety-critical objects undetectable or detected with incorrect positions. Moreover, polynomial trajectory perturbation is developed to achieve a temporally-smooth and highly-imperceptible attack. Extensive experiments on 3D object detection have shown that such attacks not only lower the performance of the state-of-the-art detectors effectively, but also transfer to other detectors, raising a red flag for the community. The code is available on https://ai4ce.github.io/FLAT/.

@article{
 title = {VIN: Voxel-based Implicit Network for Joint 3D Object Detection and Segmentation for Lidars},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.02980},
 id = {f3f19d1e-2d14-3930-9150-b1020aac9002},
 created = {2022-08-18T10:51:28.407Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:51:32.460Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {A unified neural network structure is presented for joint 3D object detection and point cloud segmentation in this paper. We leverage rich supervision from both detection and segmentation labels rather than using just one of them. In addition, an extension based on single-stage object detectors is proposed based on the implicit function widely used in 3D scene and object understanding. The extension branch takes the final feature map from the object detection module as input, and produces an implicit function that generates semantic distribution for each point for its corresponding voxel center. We demonstrated the performance of our structure on nuScenes-lidarseg, a large-scale outdoor dataset. Our solution achieves competitive results against state-of-the-art methods in both 3D object detection and point cloud segmentation with little additional computation load compared with object detection solutions. The capability of efficient weakly supervision semantic segmentation of the proposed method is also validated by experiments.},
 bibtype = {article},
 author = {Zhong, Yuanxin and Zhu, Minghan and Peng, Huei}
}

A unified neural network structure is presented for joint 3D object detection and point cloud segmentation in this paper. We leverage rich supervision from both detection and segmentation labels rather than using just one of them. In addition, an extension based on single-stage object detectors is proposed based on the implicit function widely used in 3D scene and object understanding. The extension branch takes the final feature map from the object detection module as input, and produces an implicit function that generates semantic distribution for each point for its corresponding voxel center. We demonstrated the performance of our structure on nuScenes-lidarseg, a large-scale outdoor dataset. Our solution achieves competitive results against state-of-the-art methods in both 3D object detection and point cloud segmentation with little additional computation load compared with object detection solutions. The capability of efficient weakly supervision semantic segmentation of the proposed method is also validated by experiments.

@article{
 title = {Multi-Modality Task Cascade for 3D Object Detection},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2107.04013},
 id = {89cdca43-d547-30b0-bdee-9651e5951668},
 created = {2022-08-18T10:51:28.419Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:51:35.480Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Point clouds and RGB images are naturally complementary modalities for 3D visual understanding - the former provides sparse but accurate locations of points on objects, while the latter contains dense color and texture information. Despite this potential for close sensor fusion, many methods train two models in isolation and use simple feature concatenation to represent 3D sensor data. This separated training scheme results in potentially sub-optimal performance and prevents 3D tasks from being used to benefit 2D tasks that are often useful on their own. To provide a more integrated approach, we propose a novel Multi-Modality Task Cascade network (MTC-RCNN) that leverages 3D box proposals to improve 2D segmentation predictions, which are then used to further refine the 3D boxes. We show that including a 2D network between two stages of 3D modules significantly improves both 2D and 3D task performance. Moreover, to prevent the 3D module from over-relying on the overfitted 2D predictions, we propose a dual-head 2D segmentation training and inference scheme, allowing the 2nd 3D module to learn to interpret imperfect 2D segmentation predictions. Evaluating our model on the challenging SUN RGB-D dataset, we improve upon state-of-the-art results of both single modality and fusion networks by a large margin ($\textbf+3.8$ mAP@0.5). Code will be released $\hrefhttps://github.com/Divadi/MTC_RCNN\texthere.$},
 bibtype = {article},
 author = {Park, Jinhyung and Weng, Xinshuo and Man, Yunze and Kitani, Kris}
}

Point clouds and RGB images are naturally complementary modalities for 3D visual understanding - the former provides sparse but accurate locations of points on objects, while the latter contains dense color and texture information. Despite this potential for close sensor fusion, many methods train two models in isolation and use simple feature concatenation to represent 3D sensor data. This separated training scheme results in potentially sub-optimal performance and prevents 3D tasks from being used to benefit 2D tasks that are often useful on their own. To provide a more integrated approach, we propose a novel Multi-Modality Task Cascade network (MTC-RCNN) that leverages 3D box proposals to improve 2D segmentation predictions, which are then used to further refine the 3D boxes. We show that including a 2D network between two stages of 3D modules significantly improves both 2D and 3D task performance. Moreover, to prevent the 3D module from over-relying on the overfitted 2D predictions, we propose a dual-head 2D segmentation training and inference scheme, allowing the 2nd 3D module to learn to interpret imperfect 2D segmentation predictions. Evaluating our model on the challenging SUN RGB-D dataset, we improve upon state-of-the-art results of both single modality and fusion networks by a large margin ($\textbf+3.8$ mAP@0.5). Code will be released $\hrefhttps://github.com/Divadi/MTC_RCNN\texthere.$

@article{
 title = {Self-Supervised Point Cloud Completion via Inpainting},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2111.10701},
 id = {172063bb-e8bc-3cfd-b52d-b4f569146a8e},
 created = {2022-08-18T10:53:48.475Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:09.676Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 abstract = {When navigating in urban environments, many of the objects that need to be tracked and avoided are heavily occluded. Planning and tracking using these partial scans can be challenging. The aim of this work is to learn to complete these partial point clouds, giving us a full understanding of the object's geometry using only partial observations. Previous methods achieve this with the help of complete, ground-truth annotations of the target objects, which are available only for simulated datasets. However, such ground truth is unavailable for real-world LiDAR data. In this work, we present a self-supervised point cloud completion algorithm, PointPnCNet, which is trained only on partial scans without assuming access to complete, ground-truth annotations. Our method achieves this via inpainting. We remove a portion of the input data and train the network to complete the missing region. As it is difficult to determine which regions were occluded in the initial cloud and which were synthetically removed, our network learns to complete the full cloud, including the missing regions in the initial partial cloud. We show that our method outperforms previous unsupervised and weakly-supervised methods on both the synthetic dataset, ShapeNet, and real-world LiDAR dataset, Semantic KITTI.},
 bibtype = {article},
 author = {Mittal, Himangi and Okorn, Brian and Jangid, Arpit and Held, David}
}

When navigating in urban environments, many of the objects that need to be tracked and avoided are heavily occluded. Planning and tracking using these partial scans can be challenging. The aim of this work is to learn to complete these partial point clouds, giving us a full understanding of the object's geometry using only partial observations. Previous methods achieve this with the help of complete, ground-truth annotations of the target objects, which are available only for simulated datasets. However, such ground truth is unavailable for real-world LiDAR data. In this work, we present a self-supervised point cloud completion algorithm, PointPnCNet, which is trained only on partial scans without assuming access to complete, ground-truth annotations. Our method achieves this via inpainting. We remove a portion of the input data and train the network to complete the missing region. As it is difficult to determine which regions were occluded in the initial cloud and which were synthetically removed, our network learns to complete the full cloud, including the missing regions in the initial partial cloud. We show that our method outperforms previous unsupervised and weakly-supervised methods on both the synthetic dataset, ShapeNet, and real-world LiDAR dataset, Semantic KITTI.

@article{
 title = {Adversarial Robustness Comparison of Vision Transformer and MLP-Mixer to CNNs},
 type = {article},
 year = {2021},
 pages = {1-16},
 websites = {http://arxiv.org/abs/2110.02797},
 id = {c310377b-b68d-3775-87d6-402a4f9e6208},
 created = {2022-08-18T10:53:48.488Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:53:57.679Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model architectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adversarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations.},
 bibtype = {article},
 author = {Benz, Philipp and Ham, Soomin and Zhang, Chaoning and Karjauv, Adil and Kweon, In So}
}

Convolutional Neural Networks (CNNs) have become the de facto gold standard in computer vision applications in the past years. Recently, however, new model architectures have been proposed challenging the status quo. The Vision Transformer (ViT) relies solely on attention modules, while the MLP-Mixer architecture substitutes the self-attention modules with Multi-Layer Perceptrons (MLPs). Despite their great success, CNNs have been widely known to be vulnerable to adversarial attacks, causing serious concerns for security-sensitive applications. Thus, it is critical for the community to know whether the newly proposed ViT and MLP-Mixer are also vulnerable to adversarial attacks. To this end, we empirically evaluate their adversarial robustness under several adversarial attack setups and benchmark them against the widely used CNNs. Overall, we find that the two architectures, especially ViT, are more robust than their CNN models. Using a toy example, we also provide empirical evidence that the lower adversarial robustness of CNNs can be partially attributed to their shift-invariant property. Our frequency analysis suggests that the most robust ViT architectures tend to rely more on low-frequency features compared with CNNs. Additionally, we have an intriguing finding that MLP-Mixer is extremely vulnerable to universal adversarial perturbations.

@article{
 title = {Self-Supervised Pretraining of 3D Features on any Point-Cloud},
 type = {article},
 year = {2021},
 pages = {10232-10243},
 id = {17adf16e-0e21-374d-a286-b891a8ec5982},
 created = {2022-08-18T10:53:48.615Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:46.690Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Pretraining on large labeled datasets is a prerequisite to achieve good performance in many computer vision tasks like image recognition, video understanding etc. However, pretraining is not widely used for 3D recognition tasks where state-of-the-art methods train models from scratch. A primary reason is the lack of large annotated datasets because 3D data labelling is time-consuming. Recent work shows that self-supervised learning is useful to pretrain models in 3D but requires multi-view data and point correspondences. We present a simple self-supervised pretraining method that can work with single-view depth scans acquired by varied sensors, without 3D registration and point correspondences. We pretrain standard point cloud and voxel based model architectures, and show that joint pretraining further improves performance. We evaluate our models on 9 benchmarks for object detection, semantic segmentation, and object classification, where they achieve state-of-the-art results. Most notably, we set a new state-of-the-art for object detection on ScanNet (69.0% mAP) and SUNRGBD (63.5% mAP). Our pretrained models are label efficient and improve performance for classes with few examples.},
 bibtype = {article},
 author = {Zhang, Zaiwei and Girdhar, Rohit and Joulin, Armand and Misra, Ishan},
 doi = {10.1109/ICCV48922.2021.01009},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

Pretraining on large labeled datasets is a prerequisite to achieve good performance in many computer vision tasks like image recognition, video understanding etc. However, pretraining is not widely used for 3D recognition tasks where state-of-the-art methods train models from scratch. A primary reason is the lack of large annotated datasets because 3D data labelling is time-consuming. Recent work shows that self-supervised learning is useful to pretrain models in 3D but requires multi-view data and point correspondences. We present a simple self-supervised pretraining method that can work with single-view depth scans acquired by varied sensors, without 3D registration and point correspondences. We pretrain standard point cloud and voxel based model architectures, and show that joint pretraining further improves performance. We evaluate our models on 9 benchmarks for object detection, semantic segmentation, and object classification, where they achieve state-of-the-art results. Most notably, we set a new state-of-the-art for object detection on ScanNet (69.0% mAP) and SUNRGBD (63.5% mAP). Our pretrained models are label efficient and improve performance for classes with few examples.

@article{
 title = {Imperceptible Transfer Attack and Defense on 3D Point Cloud Classification},
 type = {article},
 year = {2021},
 volume = {14},
 websites = {http://arxiv.org/abs/2111.10990},
 publisher = {IEEE},
 id = {5c1a1c6e-2055-33e4-b7d3-e93584007f46},
 created = {2022-08-18T10:53:48.620Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:07.019Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8f8f5505-1a28-42dd-a82c-92b5738465f1,13ed0c6c-c963-4099-91df-77a297ea5770,db36ed60-3b58-424a-b9a4-a9c7322975f3},
 private_publication = {false},
 abstract = {Although many efforts have been made into attack and defense on the 2D image domain in recent years, few methods explore the vulnerability of 3D models. Existing 3D attackers generally perform point-wise perturbation over point clouds, resulting in deformed structures or outliers, which is easily perceivable by humans. Moreover, their adversarial examples are generated under the white-box setting, which frequently suffers from low success rates when transferred to attack remote black-box models. In this paper, we study 3D point cloud attacks from two new and challenging perspectives by proposing a novel Imperceptible Transfer Attack (ITA): 1) Imperceptibility: we constrain the perturbation direction of each point along its normal vector of the neighborhood surface, leading to generated examples with similar geometric properties and thus enhancing the imperceptibility. 2) Transferability: we develop an adversarial transformation model to generate the most harmful distortions and enforce the adversarial examples to resist it, improving their transferability to unknown black-box models. Further, we propose to train more robust black-box 3D models to defend against such ITA attacks by learning more discriminative point cloud representations. Extensive evaluations demonstrate that our ITA attack is more imperceptible and transferable than state-of-the-arts and validate the superiority of our defense strategy.},
 bibtype = {article},
 author = {Liu, Daizong and Hu, Wei},
 doi = {10.1109/TPAMI.2022.3193449},
 number = {8}
}

Although many efforts have been made into attack and defense on the 2D image domain in recent years, few methods explore the vulnerability of 3D models. Existing 3D attackers generally perform point-wise perturbation over point clouds, resulting in deformed structures or outliers, which is easily perceivable by humans. Moreover, their adversarial examples are generated under the white-box setting, which frequently suffers from low success rates when transferred to attack remote black-box models. In this paper, we study 3D point cloud attacks from two new and challenging perspectives by proposing a novel Imperceptible Transfer Attack (ITA): 1) Imperceptibility: we constrain the perturbation direction of each point along its normal vector of the neighborhood surface, leading to generated examples with similar geometric properties and thus enhancing the imperceptibility. 2) Transferability: we develop an adversarial transformation model to generate the most harmful distortions and enforce the adversarial examples to resist it, improving their transferability to unknown black-box models. Further, we propose to train more robust black-box 3D models to defend against such ITA attacks by learning more discriminative point cloud representations. Extensive evaluations demonstrate that our ITA attack is more imperceptible and transferable than state-of-the-arts and validate the superiority of our defense strategy.

@article{
 title = {Local and Global Point Cloud Reconstruction for 3D Hand Pose Estimation},
 type = {article},
 year = {2021},
 pages = {1-15},
 websites = {http://arxiv.org/abs/2112.06389},
 id = {3b1ccf0a-b63e-3fbe-84e9-9b029fc0ec65},
 created = {2022-08-18T10:53:48.643Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:04.422Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {This paper addresses the 3D point cloud reconstruction and 3D pose estimation of the human hand from a single RGB image. To that end, we present a novel pipeline for local and global point cloud reconstruction using a 3D hand template while learning a latent representation for pose estimation. To demonstrate our method, we introduce a new multi-view hand posture dataset to obtain complete 3D point clouds of the hand in the real world. Experiments on our newly proposed dataset and four public benchmarks demonstrate the model's strengths. Our method outperforms competitors in 3D pose estimation while reconstructing realistic-looking complete 3D hand point clouds.},
 bibtype = {article},
 author = {Yu, Ziwei and Yang, Linlin and Chen, Shicheng and Yao, Angela}
}

This paper addresses the 3D point cloud reconstruction and 3D pose estimation of the human hand from a single RGB image. To that end, we present a novel pipeline for local and global point cloud reconstruction using a 3D hand template while learning a latent representation for pose estimation. To demonstrate our method, we introduce a new multi-view hand posture dataset to obtain complete 3D point clouds of the hand in the real world. Experiments on our newly proposed dataset and four public benchmarks demonstrate the model's strengths. Our method outperforms competitors in 3D pose estimation while reconstructing realistic-looking complete 3D hand point clouds.

@article{
 title = {Point-set Distances for Learning Representations of 3D Point Clouds},
 type = {article},
 year = {2021},
 pages = {10458-10467},
 id = {7599bbdc-6bd0-3ff1-86eb-ff00c24de34d},
 created = {2022-08-18T10:53:48.645Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T17:25:32.663Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {034ae31f-a548-45de-8507-3cbbc9e326ad,07e07de9-bcac-4934-a82b-d0aff540e56d},
 private_publication = {false},
 abstract = {Learning an effective representation of 3D point clouds requires a good metric to measure the discrepancy between two 3D point sets, which is non-trivial due to their irregularity. Most of the previous works resort to using the Chamfer discrepancy or Earth Mover's distance, but those metrics are either ineffective in measuring the differences between point clouds or computationally expensive. In this paper, we conduct a systematic study with extensive experiments on distance metrics for 3D point clouds. From this study, we propose to use sliced Wasserstein distance and its variants for learning representations of 3D point clouds. In addition, we introduce a new algorithm to estimate sliced Wasserstein distance that guarantees that the estimated value is close enough to the true one. Experiments show that the sliced Wasserstein distance and its variants allow the neural network to learn a more efficient representation compared to the Chamfer discrepancy. We demonstrate the efficiency of the sliced Wasserstein metric and its variants on several tasks in 3D computer vision including training a point cloud autoencoder, generative modeling, transfer learning, and point cloud registration.},
 bibtype = {article},
 author = {Nguyen, Trung and Pham, Quang Hieu and Le, Tam and Pham, Tung and Ho, Nhat and Hua, Binh Son},
 doi = {10.1109/ICCV48922.2021.01031},
 journal = {Proceedings of the IEEE International Conference on Computer Vision},
 number = {Section 4}
}

Learning an effective representation of 3D point clouds requires a good metric to measure the discrepancy between two 3D point sets, which is non-trivial due to their irregularity. Most of the previous works resort to using the Chamfer discrepancy or Earth Mover's distance, but those metrics are either ineffective in measuring the differences between point clouds or computationally expensive. In this paper, we conduct a systematic study with extensive experiments on distance metrics for 3D point clouds. From this study, we propose to use sliced Wasserstein distance and its variants for learning representations of 3D point clouds. In addition, we introduce a new algorithm to estimate sliced Wasserstein distance that guarantees that the estimated value is close enough to the true one. Experiments show that the sliced Wasserstein distance and its variants allow the neural network to learn a more efficient representation compared to the Chamfer discrepancy. We demonstrate the efficiency of the sliced Wasserstein metric and its variants on several tasks in 3D computer vision including training a point cloud autoencoder, generative modeling, transfer learning, and point cloud registration.

@article{
 title = {TSGCNet: Discriminative Geometric Feature Learning with Two-Stream Graph Convolutional Network for 3D Dental Model Segmentation},
 type = {article},
 year = {2021},
 pages = {6695-6704},
 id = {2fb60ce6-ca28-35b2-9af8-6b4a4b89a37b},
 created = {2022-08-18T10:53:48.648Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:43.999Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {The ability to segment teeth precisely from digitized 3D dental models is an essential task in computer-aided orthodontic surgical planning. To date, deep learning based methods have been popularly used to handle this task. State-of-the-art methods directly concatenate the raw attributes of 3D inputs, namely coordinates and normal vectors of mesh cells, to train a single-stream network for fully-automated tooth segmentation. This, however, has the drawback of ignoring the different geometric meanings provided by those raw attributes. This issue might possibly confuse the network in learning discriminative geometric features and result in many isolated false predictions on the dental model. Against this issue, we propose a two-stream graph convolutional network (TSGCNet) to learn multi-view geometric information from different geometric attributes. Our TSGCNet adopts two graph-learning streams, designed in an input-aware fashion, to extract more discriminative high-level geometric representations from coordinates and normal vectors, respectively. These feature representations learned from the designed two different streams are further fused to integrate the multi-view complementary information for the cell-wise dense prediction task. We evaluate our proposed TSGCNet on a real-patient dataset of dental models acquired by 3D intraoral scanners, and experimental results demonstrate that our method significantly outperforms state-of-the-art methods for 3D shape segmentation.},
 bibtype = {article},
 author = {Zhang, Lingming and Zhao, Yue and Meng, Deyu and Cui, Zhiming and Gao, Chenqiang and Gao, Xinbo and Lian, Chunfeng and Shen, Dinggang},
 doi = {10.1109/CVPR46437.2021.00663},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

The ability to segment teeth precisely from digitized 3D dental models is an essential task in computer-aided orthodontic surgical planning. To date, deep learning based methods have been popularly used to handle this task. State-of-the-art methods directly concatenate the raw attributes of 3D inputs, namely coordinates and normal vectors of mesh cells, to train a single-stream network for fully-automated tooth segmentation. This, however, has the drawback of ignoring the different geometric meanings provided by those raw attributes. This issue might possibly confuse the network in learning discriminative geometric features and result in many isolated false predictions on the dental model. Against this issue, we propose a two-stream graph convolutional network (TSGCNet) to learn multi-view geometric information from different geometric attributes. Our TSGCNet adopts two graph-learning streams, designed in an input-aware fashion, to extract more discriminative high-level geometric representations from coordinates and normal vectors, respectively. These feature representations learned from the designed two different streams are further fused to integrate the multi-view complementary information for the cell-wise dense prediction task. We evaluate our proposed TSGCNet on a real-patient dataset of dental models acquired by 3D intraoral scanners, and experimental results demonstrate that our method significantly outperforms state-of-the-art methods for 3D shape segmentation.

@article{
 title = {Dganet: A dilated graph attention-based network for local feature extraction on 3d point clouds},
 type = {article},
 year = {2021},
 keywords = {3D point clouds,Deep learning,Graph attention mechanism,Local feature extraction},
 volume = {13},
 id = {62b96fa6-6267-31ca-ac0e-991539308641},
 created = {2022-08-18T10:53:48.756Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:41.647Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Feature extraction on point clouds is an essential task when analyzing and processing point clouds of 3D scenes. However, there still remains a challenge to adequately exploit local fine-grained features on point cloud data due to its irregular and unordered structure in a 3D space. To alleviate this problem, a Dilated Graph Attention-based Network (DGANet) with a certain feature for learning ability is proposed. Specifically, we first build a local dilated graph-like region for each input point to establish the long-range spatial correlation towards its corresponding neighbors, which allows the proposed network to access a wider range of geometric information of local points with their long-range dependencies. Moreover, by integrating the dilated graph attention module (DGAM) implemented by a novel offset–attention mechanism, the proposed network promises to highlight the differing importance on each edge of the constructed local graph to uniquely learn the discrepancy feature of geometric attributes between the connected point pairs. Finally, all the learned edge attention features are further aggregated, allowing the most significant geometric feature representation of local regions by the graph–attention pooling to fully extract local detailed features for each point. The validation experiments using two challenging benchmark datasets demonstrate the effectiveness and powerful generation ability of our proposed DGANet in both 3D object classification and segmentation tasks.},
 bibtype = {article},
 author = {Wan, Jie and Xie, Zhong and Xu, Yongyang and Zeng, Ziyin and Yuan, Ding and Qiu, Qinjun},
 doi = {10.3390/rs13173484},
 journal = {Remote Sensing},
 number = {17}
}

Feature extraction on point clouds is an essential task when analyzing and processing point clouds of 3D scenes. However, there still remains a challenge to adequately exploit local fine-grained features on point cloud data due to its irregular and unordered structure in a 3D space. To alleviate this problem, a Dilated Graph Attention-based Network (DGANet) with a certain feature for learning ability is proposed. Specifically, we first build a local dilated graph-like region for each input point to establish the long-range spatial correlation towards its corresponding neighbors, which allows the proposed network to access a wider range of geometric information of local points with their long-range dependencies. Moreover, by integrating the dilated graph attention module (DGAM) implemented by a novel offset–attention mechanism, the proposed network promises to highlight the differing importance on each edge of the constructed local graph to uniquely learn the discrepancy feature of geometric attributes between the connected point pairs. Finally, all the learned edge attention features are further aggregated, allowing the most significant geometric feature representation of local regions by the graph–attention pooling to fully extract local detailed features for each point. The validation experiments using two challenging benchmark datasets demonstrate the effectiveness and powerful generation ability of our proposed DGANet in both 3D object classification and segmentation tasks.

@article{
 title = {Group-Free 3D Object Detection via Transformers},
 type = {article},
 year = {2021},
 pages = {2929-2938},
 id = {d4973e16-1792-3328-988f-3549049a3abb},
 created = {2022-08-18T10:53:48.769Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:38.733Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Recently, directly detecting 3D objects from 3D point clouds has received increasing attention. To extract object representation from an irregular point cloud, existing methods usually take a point grouping step to assign the points to an object candidate so that a PointNet-like network could be used to derive object features from the grouped points. However, the inaccurate point assignments caused by the hand-crafted grouping scheme decrease the performance of 3D object detection. In this paper, we present a simple yet effective method for directly detecting 3D objects from the 3D point cloud. Instead of grouping local points to each object candidate, our method computes the feature of an object from all the points in the point cloud with the help of an attention mechanism in the Transformers [42], where the contribution of each point is automatically learned in the network training. With an improved attention stacking scheme, our method fuses object features in different stages and generates more accurate object detection results. With few bells and whistles, the proposed method achieves state-of-the-art 3D object detection performance on two widely used benchmarks, ScanNet V2 and SUN RGB-D. The code and models are publicly available at https://github.com/zeliu98/Group-Free-3D.},
 bibtype = {article},
 author = {Liu, Ze and Zhang, Zheng and Cao, Yue and Hu, Han and Tong, Xin},
 doi = {10.1109/ICCV48922.2021.00294},
 journal = {Proceedings of the IEEE International Conference on Computer Vision}
}

Recently, directly detecting 3D objects from 3D point clouds has received increasing attention. To extract object representation from an irregular point cloud, existing methods usually take a point grouping step to assign the points to an object candidate so that a PointNet-like network could be used to derive object features from the grouped points. However, the inaccurate point assignments caused by the hand-crafted grouping scheme decrease the performance of 3D object detection. In this paper, we present a simple yet effective method for directly detecting 3D objects from the 3D point cloud. Instead of grouping local points to each object candidate, our method computes the feature of an object from all the points in the point cloud with the help of an attention mechanism in the Transformers [42], where the contribution of each point is automatically learned in the network training. With an improved attention stacking scheme, our method fuses object features in different stages and generates more accurate object detection results. With few bells and whistles, the proposed method achieves state-of-the-art 3D object detection performance on two widely used benchmarks, ScanNet V2 and SUN RGB-D. The code and models are publicly available at https://github.com/zeliu98/Group-Free-3D.

@article{
 title = {3D point cloud semantic segmentation toward large-scale unstructured agricultural scene classification},
 type = {article},
 year = {2021},
 keywords = {Deep learning,Point clouds,Scene classification,Semantic segmentation,Unstructured agricultural scene},
 pages = {106445},
 volume = {190},
 websites = {https://doi.org/10.1016/j.compag.2021.106445},
 publisher = {Elsevier B.V.},
 id = {9e04d092-64ca-397f-8697-47bb28864f05},
 created = {2022-08-18T10:53:48.843Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:18.136Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {In recent years, with the development of computer vision, deep learning, and artificial intelligence technologies, the popularity of depth sensors and lidar has promoted the rapid development of three-dimensional (3D) point cloud semantic segmentation. The semantic segmentation of 3D point clouds for large-scale unstructured agricultural scenes is important for agricultural robots to perceive their surrounding environment, and for autonomous navigation and positioning and autonomous scene understanding. In this study, the problem of 3D point cloud semantic segmentation for large-scale unstructured agricultural scenes was studied. By improving the neural network structure of RandLA-Net, a deeper 3D point cloud semantic segmentation neural network model for large-scale unstructured agricultural scenes was built, and good experimental results were obtained. The local feature aggregation module in RandLA-Net was integrated and improved to achieve 3D point cloud semantic segmentation for large-scale unstructured agricultural scenes. To test the influence of the 3D point cloud sampling algorithm on the overall accuracy (OA) and mean intersection-over-union (mIoU) of semantic segmentation, the random sampling algorithm and farthest point sampling algorithm were used to build two models with the same neural network structure. The test results show that the sampling algorithm has little effect on the OA and mIoU of 3D point cloud semantic segmentation, and the final result depends mainly on the extraction of 3D point cloud features. In addition, two different Semantic3D datasets were used to test the effect of the datasets on the generalization ability of the model, and the results showed that the datasets had an important effect on the neural network model.},
 bibtype = {article},
 author = {Chen, Yi and Xiong, Yingjun and Zhang, Baohua and Zhou, Jun and Zhang, Qian},
 doi = {10.1016/j.compag.2021.106445},
 journal = {Computers and Electronics in Agriculture},
 number = {August}
}

In recent years, with the development of computer vision, deep learning, and artificial intelligence technologies, the popularity of depth sensors and lidar has promoted the rapid development of three-dimensional (3D) point cloud semantic segmentation. The semantic segmentation of 3D point clouds for large-scale unstructured agricultural scenes is important for agricultural robots to perceive their surrounding environment, and for autonomous navigation and positioning and autonomous scene understanding. In this study, the problem of 3D point cloud semantic segmentation for large-scale unstructured agricultural scenes was studied. By improving the neural network structure of RandLA-Net, a deeper 3D point cloud semantic segmentation neural network model for large-scale unstructured agricultural scenes was built, and good experimental results were obtained. The local feature aggregation module in RandLA-Net was integrated and improved to achieve 3D point cloud semantic segmentation for large-scale unstructured agricultural scenes. To test the influence of the 3D point cloud sampling algorithm on the overall accuracy (OA) and mean intersection-over-union (mIoU) of semantic segmentation, the random sampling algorithm and farthest point sampling algorithm were used to build two models with the same neural network structure. The test results show that the sampling algorithm has little effect on the OA and mIoU of 3D point cloud semantic segmentation, and the final result depends mainly on the extraction of 3D point cloud features. In addition, two different Semantic3D datasets were used to test the effect of the datasets on the generalization ability of the model, and the results showed that the datasets had an important effect on the neural network model.

@article{
 title = {Airborne LiDAR point cloud classification with global-local graph attention convolution neural network},
 type = {article},
 year = {2021},
 keywords = {Airborne LiDAR,Graph attention convolution,ISPRS 3D labeling,Point cloud classification,Point cloud deep learning},
 pages = {181-194},
 volume = {173},
 websites = {https://doi.org/10.1016/j.isprsjprs.2021.01.007},
 publisher = {Elsevier B.V.},
 id = {d294c11e-c819-3ede-a1f8-b5e66440c279},
 created = {2022-08-18T10:53:48.876Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-08-18T10:54:23.513Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Airborne light detection and ranging (LiDAR) plays an increasingly significant role in urban planning, topographic mapping, environmental monitoring, power line detection and other fields thanks to its capability to quickly acquire large-scale and high-precision ground information. To achieve point cloud classification, previous studies proposed point cloud deep learning models that can directly process raw point clouds based on PointNet-like architectures. And some recent works proposed graph convolution neural network based on the inherent topology of point clouds. However, the above point cloud deep learning models only pay attention to exploring local geometric structures, yet ignore global contextual relationships among all points. In this paper, we present a global-local graph attention convolution neural network (GACNN) that can be directly applied to the classification of unstructured 3D point clouds obtained by airborne LiDAR. Specifically, we first introduce a graph attention convolution module that incorporates global contextual information and local structural features. The global attention module examines spatial relationships among all points, while the local attention module can dynamically learn convolution weights with regard to the spatial position of the local neighboring points and reweight the convolution weights by inspecting the density of each local region. Based on the proposed graph attention convolution module, we further design an end-to-end encoder-decoder network, named GACNN, to capture multiscale features of the point clouds and therefore enable more accurate airborne point cloud classification. Experiments on the ISPRS 3D labeling dataset show that the proposed model achieves a new state-of-the-art performance in terms of average F1 score (71.5%) and a satisfying overall accuracy (83.2%). Additionally, experiments further conducted on the 2019 Data Fusion Contest Dataset by comparing with other prevalent point cloud deep learning models demonstrate the favorable generalization capability of the proposed model.},
 bibtype = {article},
 author = {Wen, Congcong and Li, Xiang and Yao, Xiaojing and Peng, Ling and Chi, Tianhe},
 doi = {10.1016/j.isprsjprs.2021.01.007},
 journal = {ISPRS Journal of Photogrammetry and Remote Sensing},
 number = {January}
}

Airborne light detection and ranging (LiDAR) plays an increasingly significant role in urban planning, topographic mapping, environmental monitoring, power line detection and other fields thanks to its capability to quickly acquire large-scale and high-precision ground information. To achieve point cloud classification, previous studies proposed point cloud deep learning models that can directly process raw point clouds based on PointNet-like architectures. And some recent works proposed graph convolution neural network based on the inherent topology of point clouds. However, the above point cloud deep learning models only pay attention to exploring local geometric structures, yet ignore global contextual relationships among all points. In this paper, we present a global-local graph attention convolution neural network (GACNN) that can be directly applied to the classification of unstructured 3D point clouds obtained by airborne LiDAR. Specifically, we first introduce a graph attention convolution module that incorporates global contextual information and local structural features. The global attention module examines spatial relationships among all points, while the local attention module can dynamically learn convolution weights with regard to the spatial position of the local neighboring points and reweight the convolution weights by inspecting the density of each local region. Based on the proposed graph attention convolution module, we further design an end-to-end encoder-decoder network, named GACNN, to capture multiscale features of the point clouds and therefore enable more accurate airborne point cloud classification. Experiments on the ISPRS 3D labeling dataset show that the proposed model achieves a new state-of-the-art performance in terms of average F1 score (71.5%) and a satisfying overall accuracy (83.2%). Additionally, experiments further conducted on the 2019 Data Fusion Contest Dataset by comparing with other prevalent point cloud deep learning models demonstrate the favorable generalization capability of the proposed model.

@article{
 title = {FinerPCN: High fidelity point cloud completion network using pointwise convolution},
 type = {article},
 year = {2021},
 keywords = {3D point cloud,Deep learning,Point analysis,Point completion network,Shape completion},
 pages = {266-276},
 volume = {460},
 websites = {https://doi.org/10.1016/j.neucom.2021.06.080},
 publisher = {Elsevier},
 id = {6ed47f61-aa8a-3938-9e06-e1873bc072a1},
 created = {2022-08-18T10:53:48.876Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:09.514Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 abstract = {3D scanners often obtain partial point clouds due to occlusion and limitation of viewing angles. Point cloud completion aims at inferring the full shape of an object from an incomplete point set. Existing deep learning models either do not consider local information or easily degrade the sharp details of the input, thereby losing some existing structures. In this paper, we propose a high fidelity point cloud completion network using pointwise convolution, called FinerPCN. FinerPCN generates complete and fine point clouds in a coarse-to-fine manner. FinerPCN consists of two subnetworks: an encoder-decoder for generating a coarse shape and pointwise convolution for refining its local structure. By repeatedly feeding partial input into the second subnetwork, FinerPCN effectively considers local information and alleviates structural blur of input while maintaining global shape. Experimental results show that FinerPCN generates finer detailed completion results than state-of-the-art methods while successfully keeping the shape of the input.},
 bibtype = {article},
 author = {Chang, Yakun and Jung, Cheolkon and Xu, Yuanquan},
 doi = {10.1016/j.neucom.2021.06.080},
 journal = {Neurocomputing}
}

3D scanners often obtain partial point clouds due to occlusion and limitation of viewing angles. Point cloud completion aims at inferring the full shape of an object from an incomplete point set. Existing deep learning models either do not consider local information or easily degrade the sharp details of the input, thereby losing some existing structures. In this paper, we propose a high fidelity point cloud completion network using pointwise convolution, called FinerPCN. FinerPCN generates complete and fine point clouds in a coarse-to-fine manner. FinerPCN consists of two subnetworks: an encoder-decoder for generating a coarse shape and pointwise convolution for refining its local structure. By repeatedly feeding partial input into the second subnetwork, FinerPCN effectively considers local information and alleviates structural blur of input while maintaining global shape. Experimental results show that FinerPCN generates finer detailed completion results than state-of-the-art methods while successfully keeping the shape of the input.

@article{
 title = {HSGAN: Hierarchical Graph Learning for Point Cloud Generation},
 type = {article},
 year = {2021},
 keywords = {GAN,Generative Adversarial Network,gradient penalty,graph learning,machine learning,point cloud generation,self-attention},
 pages = {4540-4554},
 volume = {30},
 id = {54f4f79c-9111-39e5-899f-1bf7f6391293},
 created = {2022-08-18T10:53:48.893Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-13T08:14:28.761Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {8f8f5505-1a28-42dd-a82c-92b5738465f1,b6d75013-efe2-4ddc-b3db-65496bd4db9f,a6a80a30-e9a2-486d-8032-eac3fd981996,df1ae3c5-d972-4eac-9fff-d018bc64f4a9,244f8db2-6bd4-47d9-8abf-425a263fd4d1},
 private_publication = {false},
 abstract = {Point clouds are the most general data representations of real and abstract objects, and have a wide variety of applications in many science and engineering fields. Point clouds also provide the most scalable multi-resolution composition for geometric structures. Although point cloud learning has shown remarkable results in shape estimation and semantic segmentation, the unsupervised generation of 3D object parts still pose significant challenges in the 3D shape understanding problem. We address this problem by proposing a novel Generative Adversarial Network (GAN), named HSGAN, or Hierarchical Self-Attention GAN, with remarkable properties for 3D shape generation. Our generative model takes a random code and hierarchically transforms it into a representation graph by incorporating both Graph Convolution Network (GCN) and self-attention. With embedding the global graph topology in shape generation, the proposed model takes advantage of the latent topological information to fully construct the geometry of 3D object shapes. Different from the existing generative pipelines, our deep learning architecture articulates three significant properties HSGAN effectively deploys the compact latent topology information as a graph representation in the generative learning process and generates realistic point clouds, HSGAN avoids multiple discriminator updates per generator update, and HSGAN preserves the most dominant geometric structures of 3D shapes in the same hierarchical sampling process. We demonstrate the performance of our new approach with both quantitative and qualitative evaluations. We further present a new adversarial loss to maintain the training stability and overcome the potential mode collapse of traditional GANs. Finally, we explore the use of HSGAN as a plug-and-play decoder in the auto-encoding architecture.},
 bibtype = {article},
 author = {Li, Yushi and Baciu, George},
 doi = {10.1109/TIP.2021.3073318},
 journal = {IEEE Transactions on Image Processing}
}

Point clouds are the most general data representations of real and abstract objects, and have a wide variety of applications in many science and engineering fields. Point clouds also provide the most scalable multi-resolution composition for geometric structures. Although point cloud learning has shown remarkable results in shape estimation and semantic segmentation, the unsupervised generation of 3D object parts still pose significant challenges in the 3D shape understanding problem. We address this problem by proposing a novel Generative Adversarial Network (GAN), named HSGAN, or Hierarchical Self-Attention GAN, with remarkable properties for 3D shape generation. Our generative model takes a random code and hierarchically transforms it into a representation graph by incorporating both Graph Convolution Network (GCN) and self-attention. With embedding the global graph topology in shape generation, the proposed model takes advantage of the latent topological information to fully construct the geometry of 3D object shapes. Different from the existing generative pipelines, our deep learning architecture articulates three significant properties HSGAN effectively deploys the compact latent topology information as a graph representation in the generative learning process and generates realistic point clouds, HSGAN avoids multiple discriminator updates per generator update, and HSGAN preserves the most dominant geometric structures of 3D shapes in the same hierarchical sampling process. We demonstrate the performance of our new approach with both quantitative and qualitative evaluations. We further present a new adversarial loss to maintain the training stability and overcome the potential mode collapse of traditional GANs. Finally, we explore the use of HSGAN as a plug-and-play decoder in the auto-encoding architecture.

@article{
 title = {Real-Time Volumetric-Semantic Exploration and Mapping : An Uncertainty-Aware Approach},
 type = {article},
 year = {2021},
 pages = {9064-9070},
 publisher = {IEEE},
 id = {fe70cc72-29b1-3cd4-a945-cdad3f983893},
 created = {2022-08-19T11:38:34.703Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:09.839Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 bibtype = {article},
 author = {Figueiredo, Rui Pimentel De and Sejersen, Fevre and Hansen, Jakob Grimm and Brand, Martim}
}

@article{
 title = {Learning Graph Representation with Generative Adversarial Nets},
 type = {article},
 year = {2021},
 keywords = {Graph representation learning,generative adversarial nets,graph softmax},
 pages = {3090-3103},
 volume = {33},
 id = {5eb5a40d-da22-34e3-b6d2-c66d7803393f},
 created = {2022-08-19T12:56:19.270Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:03.415Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 private_publication = {false},
 abstract = {Graph representation learning aims to embed each vertex in a graph into a low-dimensional vector space. Existing graph representation learning methods can be classified into two categories: generative models that learn the underlying connectivity distribution in a graph, and discriminative models that predict the probability of edge between a pair of vertices. In this paper, we propose GraphGAN, an innovative graph representation learning framework unifying the above two classes of methods, in which the generative and the discriminative model play a game-Theoretical minimax game. Specifically, for a given vertex, the generative model tries to fit its underlying true connectivity distribution over all other vertices and produces 'fake' samples to fool the discriminative model, while the discriminative model tries to detect whether the sampled vertex is from ground truth or generated by the generative model. With the competition between these two models, both of them can alternately and iteratively boost their performance. Moreover, we propose a novel graph softmax as the implementation of the generative model to overcome the limitations of traditional softmax function, which can be proven satisfying desirable properties of normalization, graph structure awareness, and computational efficiency. Through extensive experiments on real-world datasets, we demonstrate that GraphGAN achieves substantial gains in a variety of applications, including graph reconstruction, link prediction, node classification, recommendation, and visualization, over state-of-The-Art baselines.},
 bibtype = {article},
 author = {Wang, Hongwei and Wang, Jialin and Wang, Jia and Zhao, Miao and Zhang, Weinan and Zhang, Fuzheng and Li, Wenjie and Xie, Xing and Guo, Minyi},
 doi = {10.1109/TKDE.2019.2961882},
 journal = {IEEE Transactions on Knowledge and Data Engineering},
 number = {8}
}

Graph representation learning aims to embed each vertex in a graph into a low-dimensional vector space. Existing graph representation learning methods can be classified into two categories: generative models that learn the underlying connectivity distribution in a graph, and discriminative models that predict the probability of edge between a pair of vertices. In this paper, we propose GraphGAN, an innovative graph representation learning framework unifying the above two classes of methods, in which the generative and the discriminative model play a game-Theoretical minimax game. Specifically, for a given vertex, the generative model tries to fit its underlying true connectivity distribution over all other vertices and produces 'fake' samples to fool the discriminative model, while the discriminative model tries to detect whether the sampled vertex is from ground truth or generated by the generative model. With the competition between these two models, both of them can alternately and iteratively boost their performance. Moreover, we propose a novel graph softmax as the implementation of the generative model to overcome the limitations of traditional softmax function, which can be proven satisfying desirable properties of normalization, graph structure awareness, and computational efficiency. Through extensive experiments on real-world datasets, we demonstrate that GraphGAN achieves substantial gains in a variety of applications, including graph reconstruction, link prediction, node classification, recommendation, and visualization, over state-of-The-Art baselines.

@article{
 title = {Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model},
 type = {article},
 year = {2021},
 pages = {1-21},
 id = {7dccf841-0a6c-31c7-944a-784b40039423},
 created = {2022-09-06T11:37:21.054Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:10.051Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 bibtype = {article},
 author = {Schrittwieser, Julian and Antonoglou, Ioannis and Hubert, Thomas and Simonyan, Karen and Sifre, Laurent and Schmitt, Simon and Guez, Arthur and Lockhart, Edward and Hassabis, Demis and Graepel, Thore and Lillicrap, Timothy}
}

@article{
 title = {Real-Time Volumetric-Semantic Exploration and Mapping : An Uncertainty-Aware Approach},
 type = {article},
 year = {2021},
 pages = {9064-9070},
 publisher = {IEEE},
 id = {1017349e-d755-30ec-9c2e-18fa53355aab},
 created = {2022-09-06T11:37:21.072Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-06T11:37:25.824Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 private_publication = {false},
 bibtype = {article},
 author = {Figueiredo, Rui Pimentel De and Sejersen, Fevre and Hansen, Jakob Grimm and Brand, Martim}
}

@article{
 title = {Next-best-view regression using a 3D convolutional neural network},
 type = {article},
 year = {2021},
 keywords = {3D modeling,Deep learning,Next-best-view,Object reconstruction,Range sensing},
 pages = {1-14},
 volume = {32},
 websites = {https://doi.org/10.1007/s00138-020-01166-2},
 publisher = {Springer Berlin Heidelberg},
 id = {49c1967c-1f8f-35a4-b05d-c90239267586},
 created = {2022-09-08T11:24:59.063Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-08T11:25:19.245Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {6d08e913-c810-48b8-98aa-c6e35d83017e,2a64087a-cd32-494a-8140-2abf0b1356c6},
 private_publication = {false},
 abstract = {Automated three-dimensional (3D) object reconstruction is the task of building a geometric representation of a physical object by means of sensing its surface. Even though new single-view reconstruction techniques can predict the surface, they lead to incomplete models, specially, for non-commons objects such as antique objects or art sculptures. Therefore, to achieve the task’s goals, it is essential to automatically determine the locations where the sensor will be placed so that the surface will be completely observed. This problem is known as the next-best-view problem. In this paper, we propose a data-driven approach to address the problem. The proposed approach trains a 3D convolutional neural network (3D CNN) with previous reconstructions in order to regress the position of the next-best-view. To the best of our knowledge, this is one of the first works that directly infers the next-best-view in a continuous space using a data-driven approach for the 3D object reconstruction task. We have validated the proposed approach making use of two groups of experiments. In the first group, several variants of the proposed architecture are analyzed. Predicted next-best-views were observed to be closely positioned to the ground truth. In the second group of experiments, the proposed approach is requested to reconstruct several unseen objects, namely, objects not considered by the 3D CNN during training nor validation. Coverage percentages of up to 90 % were observed. With respect to current state-of-the-art methods, the proposed approach improves the performance of previous next-best-view classification approaches and it is quite fast in running time (3 frames per second), given that it does not compute the expensive ray tracing required by previous information metrics.},
 bibtype = {article},
 author = {Vasquez-Gomez, J. Irving and Troncoso, David and Becerra, Israel and Sucar, Enrique and Murrieta-Cid, Rafael},
 doi = {10.1007/s00138-020-01166-2},
 journal = {Machine Vision and Applications},
 number = {2}
}

Automated three-dimensional (3D) object reconstruction is the task of building a geometric representation of a physical object by means of sensing its surface. Even though new single-view reconstruction techniques can predict the surface, they lead to incomplete models, specially, for non-commons objects such as antique objects or art sculptures. Therefore, to achieve the task’s goals, it is essential to automatically determine the locations where the sensor will be placed so that the surface will be completely observed. This problem is known as the next-best-view problem. In this paper, we propose a data-driven approach to address the problem. The proposed approach trains a 3D convolutional neural network (3D CNN) with previous reconstructions in order to regress the position of the next-best-view. To the best of our knowledge, this is one of the first works that directly infers the next-best-view in a continuous space using a data-driven approach for the 3D object reconstruction task. We have validated the proposed approach making use of two groups of experiments. In the first group, several variants of the proposed architecture are analyzed. Predicted next-best-views were observed to be closely positioned to the ground truth. In the second group of experiments, the proposed approach is requested to reconstruct several unseen objects, namely, objects not considered by the 3D CNN during training nor validation. Coverage percentages of up to 90 % were observed. With respect to current state-of-the-art methods, the proposed approach improves the performance of previous next-best-view classification approaches and it is quite fast in running time (3 frames per second), given that it does not compute the expensive ray tracing required by previous information metrics.

@article{
 title = {SE-MD: A Single-encoder multiple-decoder deep network for point cloud generation from 2D images},
 type = {article},
 year = {2021},
 keywords = {2d images,3d convolutional,3d model reconstruction,3d shape generation,point clouds,shapenet},
 websites = {http://arxiv.org/abs/2106.15325},
 id = {d7f4f9b6-bd85-3082-9b76-a601e02bc716},
 created = {2022-09-13T13:57:40.424Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-09-23T10:14:51.115Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {255910b9-b737-4c31-858e-6de1dca0cdb9,a6a80a30-e9a2-486d-8032-eac3fd981996},
 private_publication = {false},
 abstract = {3D model generation from single 2D RGB images is a challenging and actively researched computer vision task. Various techniques using conventional network architectures have been proposed for the same. However, the body of research work is limited and there are various issues like using inefficient 3D representation formats, weak 3D model generation backbones, inability to generate dense point clouds, dependence of post-processing for generation of dense point clouds, and dependence on silhouettes in RGB images. In this paper, a novel 2D RGB image to point cloud conversion technique is proposed, which improves the state of art in the field due to its efficient, robust and simple model by using the concept of parallelization in network architecture. It not only uses the efficient and rich 3D representation of point clouds, but also uses a novel and robust point cloud generation backbone in order to address the prevalent issues. This involves using a single-encoder multiple-decoder deep network architecture wherein each decoder generates certain fixed viewpoints. This is followed by fusing all the viewpoints to generate a dense point cloud. Various experiments are conducted on the technique and its performance is compared with those of other state of the art techniques and impressive gains in performance are demonstrated. Code is available at https://github.com/mueedhafiz1982/},
 bibtype = {article},
 author = {Hafiz, Abdul Mueed and Bhat, Rouf Ul Alam and Parah, Shabir Ahmad and Hassaballah, M.}
}

3D model generation from single 2D RGB images is a challenging and actively researched computer vision task. Various techniques using conventional network architectures have been proposed for the same. However, the body of research work is limited and there are various issues like using inefficient 3D representation formats, weak 3D model generation backbones, inability to generate dense point clouds, dependence of post-processing for generation of dense point clouds, and dependence on silhouettes in RGB images. In this paper, a novel 2D RGB image to point cloud conversion technique is proposed, which improves the state of art in the field due to its efficient, robust and simple model by using the concept of parallelization in network architecture. It not only uses the efficient and rich 3D representation of point clouds, but also uses a novel and robust point cloud generation backbone in order to address the prevalent issues. This involves using a single-encoder multiple-decoder deep network architecture wherein each decoder generates certain fixed viewpoints. This is followed by fusing all the viewpoints to generate a dense point cloud. Various experiments are conducted on the technique and its performance is compared with those of other state of the art techniques and impressive gains in performance are demonstrated. Code is available at https://github.com/mueedhafiz1982/

@article{
 title = {PU-GCN: Point Cloud Upsampling using Graph Convolutional Networks},
 type = {article},
 year = {2021},
 pages = {11678-11687},
 id = {300e8555-d5c5-379d-a678-fe6667e901ac},
 created = {2022-09-19T07:34:03.592Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-10-03T13:31:10.334Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {b6d75013-efe2-4ddc-b3db-65496bd4db9f,a6a80a30-e9a2-486d-8032-eac3fd981996},
 private_publication = {false},
 abstract = {The effectiveness of learning-based point cloud upsampling pipelines heavily relies on the upsampling modules and feature extractors used therein. For the point upsampling module, we propose a novel model called NodeShuffle, which uses a Graph Convolutional Network (GCN) to better encode local point information from point neighborhoods. NodeShuffle is versatile and can be incorporated into any point cloud upsampling pipeline. Extensive experiments show how NodeShuffle consistently improves state-of-the-art upsampling methods. For feature extraction, we also propose a new multi-scale point feature extractor, called Inception DenseGCN. By aggregating features at multiple scales, this feature extractor enables further performance gain in the final upsampled point clouds. We combine Inception DenseGCN with NodeShuffle into a new point upsampling pipeline called PU-GCN. PU-GCN sets new state-of-art performance with much fewer parameters and more efficient inference. Our code is publicly available at https://github.com/guochengqian/PU-GCN.},
 bibtype = {article},
 author = {Qian, Guocheng and Abualshour, Abdulellah and Li, Guohao and Thabet, Ali and Ghanem, Bernard},
 doi = {10.1109/CVPR46437.2021.01151},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

The effectiveness of learning-based point cloud upsampling pipelines heavily relies on the upsampling modules and feature extractors used therein. For the point upsampling module, we propose a novel model called NodeShuffle, which uses a Graph Convolutional Network (GCN) to better encode local point information from point neighborhoods. NodeShuffle is versatile and can be incorporated into any point cloud upsampling pipeline. Extensive experiments show how NodeShuffle consistently improves state-of-the-art upsampling methods. For feature extraction, we also propose a new multi-scale point feature extractor, called Inception DenseGCN. By aggregating features at multiple scales, this feature extractor enables further performance gain in the final upsampled point clouds. We combine Inception DenseGCN with NodeShuffle into a new point upsampling pipeline called PU-GCN. PU-GCN sets new state-of-art performance with much fewer parameters and more efficient inference. Our code is publicly available at https://github.com/guochengqian/PU-GCN.

@article{
 title = {Learning Progressive Point Embeddings for 3D Point Cloud Generation},
 type = {article},
 year = {2021},
 pages = {10261-10270},
 id = {d0c5a913-61e2-34c9-b08d-4f186b9c5d9e},
 created = {2022-09-21T09:29:20.057Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-10-03T13:31:10.260Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {b6d75013-efe2-4ddc-b3db-65496bd4db9f,a6a80a30-e9a2-486d-8032-eac3fd981996},
 private_publication = {false},
 abstract = {Generative models for 3D point clouds are extremely important for scene/object reconstruction applications in autonomous driving and robotics. Despite recent success of deep learning-based representation learning, it remains a great challenge for deep neural networks to synthesize or reconstruct high-fidelity point clouds, because of the difficulties in 1) learning effective pointwise representations; and 2) generating realistic point clouds from complex distributions. In this paper, we devise a dual-generators framework for point cloud generation, which generalizes vanilla generative adversarial learning framework in a progressive manner. Specifically, the first generator aims to learn effective point embeddings in a breadth-first manner, while the second generator is used to refine the generated point cloud based on a depth-first point embedding to generate a robust and uniform point cloud. The proposed dual-generators framework thus is able to progressively learn effective point embeddings for accurate point cloud generation. Experimental results on a variety of object categories from the most popular point cloud generation dataset, ShapeNet, demonstrate the state-of-the-art performance of the proposed method for accurate point cloud generation.},
 bibtype = {article},
 author = {Wen, Cheng and Yu, Baosheng and Tao, Dacheng},
 doi = {10.1109/CVPR46437.2021.01013},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

Generative models for 3D point clouds are extremely important for scene/object reconstruction applications in autonomous driving and robotics. Despite recent success of deep learning-based representation learning, it remains a great challenge for deep neural networks to synthesize or reconstruct high-fidelity point clouds, because of the difficulties in 1) learning effective pointwise representations; and 2) generating realistic point clouds from complex distributions. In this paper, we devise a dual-generators framework for point cloud generation, which generalizes vanilla generative adversarial learning framework in a progressive manner. Specifically, the first generator aims to learn effective point embeddings in a breadth-first manner, while the second generator is used to refine the generated point cloud based on a depth-first point embedding to generate a robust and uniform point cloud. The proposed dual-generators framework thus is able to progressively learn effective point embeddings for accurate point cloud generation. Experimental results on a variety of object categories from the most popular point cloud generation dataset, ShapeNet, demonstrate the state-of-the-art performance of the proposed method for accurate point cloud generation.

@article{
 title = {Image-based 3d object reconstruction: State-of-the-art and trends in the deep learning era},
 type = {article},
 year = {2021},
 keywords = {3D face,3D human body,3D reconstruction,3D video,CNN,LSTM,SLAM,SfM,deep learning,depth estimation},
 pages = {1578-1604},
 volume = {43},
 id = {623d9908-eec3-3466-a73f-3c742baebc92},
 created = {2022-10-03T13:31:09.949Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-12-05T14:13:52.460Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e07bebd1-ae76-40ed-b298-edc5ed896e0b},
 private_publication = {false},
 abstract = {3D reconstruction is a longstanding ill-posed problem, which has been explored for decades by the computer vision, computer graphics, and machine learning communities. Since 2015, image-based 3D reconstruction using convolutional neural networks (CNN) has attracted increasing interest and demonstrated an impressive performance. Given this new era of rapid evolution, this article provides a comprehensive survey of the recent developments in this field. We focus on the works which use deep learning techniques to estimate the 3D shape of generic objects either from a single or multiple RGB images. We organize the literature based on the shape representations, the network architectures, and the training mechanisms they use. While this survey is intended for methods which reconstruct generic objects, we also review some of the recent works which focus on specific object classes such as human body shapes and faces. We provide an analysis and comparison of the performance of some key papers, summarize some of the open problems in this field, and discuss promising directions for future research.},
 bibtype = {article},
 author = {Han, Xian Feng and Laga, Hamid and Bennamoun, Mohammed},
 doi = {10.1109/TPAMI.2019.2954885},
 journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
 number = {5}
}

3D reconstruction is a longstanding ill-posed problem, which has been explored for decades by the computer vision, computer graphics, and machine learning communities. Since 2015, image-based 3D reconstruction using convolutional neural networks (CNN) has attracted increasing interest and demonstrated an impressive performance. Given this new era of rapid evolution, this article provides a comprehensive survey of the recent developments in this field. We focus on the works which use deep learning techniques to estimate the 3D shape of generic objects either from a single or multiple RGB images. We organize the literature based on the shape representations, the network architectures, and the training mechanisms they use. While this survey is intended for methods which reconstruct generic objects, we also review some of the recent works which focus on specific object classes such as human body shapes and faces. We provide an analysis and comparison of the performance of some key papers, summarize some of the open problems in this field, and discuss promising directions for future research.

@article{
 title = {Single image 3D object reconstruction based on deep learning: A review},
 type = {article},
 year = {2021},
 keywords = {3D shape representation,Computer vision,Deep learning,Single image 3D reconstruction},
 pages = {463-498},
 volume = {80},
 publisher = {Multimedia Tools and Applications},
 id = {7a923fe2-3fd6-3701-8333-1fed11fef89f},
 created = {2022-10-03T13:31:10.088Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2022-10-03T13:31:24.321Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e07bebd1-ae76-40ed-b298-edc5ed896e0b},
 private_publication = {false},
 abstract = {The reconstruction of 3D object from a single image is an important task in the field of computer vision. In recent years, 3D reconstruction of single image using deep learning technology has achieved remarkable results. Traditional methods to reconstruct 3D object from a single image require prior knowledge and assumptions, and the reconstruction object is limited to a certain category or it is difficult to accomplish a good reconstruction from a real image. Although deep learning can solve these problems well with its own powerful learning ability, it also faces many problems. In this paper, we first discuss the challenges faced by applying the deep learning method to reconstruct 3D objects from a single image. Second, we comprehensively review encoders, decoders and training details used in 3D reconstruction of a single image. Then, the common datasets and evaluation metrics of single image 3D object reconstruction in recent years are introduced. In order to analyze the advantages and disadvantages of different 3D reconstruction methods, a series of experiments are used for comparison. In addition, we simply give some related application examples involving 3D reconstruction of a single image. Finally, we summarize this paper and discuss the future directions.},
 bibtype = {article},
 author = {Fu, Kui and Peng, Jiansheng and He, Qiwen and Zhang, Hanxiao},
 doi = {10.1007/s11042-020-09722-8},
 journal = {Multimedia Tools and Applications},
 number = {1}
}

The reconstruction of 3D object from a single image is an important task in the field of computer vision. In recent years, 3D reconstruction of single image using deep learning technology has achieved remarkable results. Traditional methods to reconstruct 3D object from a single image require prior knowledge and assumptions, and the reconstruction object is limited to a certain category or it is difficult to accomplish a good reconstruction from a real image. Although deep learning can solve these problems well with its own powerful learning ability, it also faces many problems. In this paper, we first discuss the challenges faced by applying the deep learning method to reconstruct 3D objects from a single image. Second, we comprehensively review encoders, decoders and training details used in 3D reconstruction of a single image. Then, the common datasets and evaluation metrics of single image 3D object reconstruction in recent years are introduced. In order to analyze the advantages and disadvantages of different 3D reconstruction methods, a series of experiments are used for comparison. In addition, we simply give some related application examples involving 3D reconstruction of a single image. Finally, we summarize this paper and discuss the future directions.

@article{
 title = {Leveraging SE(3) Equivariance for Self-Supervised Category-Level Object Pose Estimation},
 type = {article},
 year = {2021},
 pages = {15370-15381},
 volume = {19},
 id = {b5288764-6542-3a88-8fe4-eec9ee93e6c7},
 created = {2023-05-03T13:16:38.933Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.306Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Li2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Category-level object pose estimation aims to find 6D object poses of previously unseen object instances from known categories without access to object CAD models. To reduce the huge amount of pose annotations needed for categorylevel learning, we propose for the first time a self-supervised learning framework to estimate category-level 6D object pose from single 3D point clouds. During training, our method assumes no ground-truth pose annotations, no CAD models, and no multi-view supervision. The key to our method is to disentangle shape and pose through an invariant shape reconstruction module and an equivariant pose estimation module, empowered by SE(3) equivariant point cloud networks. The invariant shape reconstruction module learns to perform aligned reconstructions, yielding a category-level reference frame without using any annotations. In addition, the equivariant pose estimation module achieves category-level pose estimation accuracy that is comparable to some fully supervised methods. Extensive experiments demonstrate the effectiveness of our approach on both complete and partial depth point clouds from the ModelNet40 benchmark, and on real depth point clouds from the NOCS-REAL 275 dataset. The project page with code and visualizations can be found at: dragonlong.github.io/equi-pose.},
 bibtype = {article},
 author = {Li, Xiaolong and Weng, Yijia and Yi, Li and Guibas, Leonidas and Abbott, A. Lynn and Song, Shuran and Wang, He},
 journal = {Advances in Neural Information Processing Systems},
 number = {3}
}

Category-level object pose estimation aims to find 6D object poses of previously unseen object instances from known categories without access to object CAD models. To reduce the huge amount of pose annotations needed for categorylevel learning, we propose for the first time a self-supervised learning framework to estimate category-level 6D object pose from single 3D point clouds. During training, our method assumes no ground-truth pose annotations, no CAD models, and no multi-view supervision. The key to our method is to disentangle shape and pose through an invariant shape reconstruction module and an equivariant pose estimation module, empowered by SE(3) equivariant point cloud networks. The invariant shape reconstruction module learns to perform aligned reconstructions, yielding a category-level reference frame without using any annotations. In addition, the equivariant pose estimation module achieves category-level pose estimation accuracy that is comparable to some fully supervised methods. Extensive experiments demonstrate the effectiveness of our approach on both complete and partial depth point clouds from the ModelNet40 benchmark, and on real depth point clouds from the NOCS-REAL 275 dataset. The project page with code and visualizations can be found at: dragonlong.github.io/equi-pose.

@article{
 title = {Spectral spherical harmonics discrete ordinate method},
 type = {article},
 year = {2021},
 keywords = {Gaussian beam,SHDOM,three-dimensional radiative trasnfer models},
 pages = {107386},
 volume = {258},
 websites = {https://doi.org/10.1016/j.jqsrt.2020.107386},
 publisher = {Elsevier Ltd},
 id = {8c2d3e9e-070e-3723-b9a5-04774f8d2ffd},
 created = {2023-05-03T13:16:38.937Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.237Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Doicu2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {A new method for modeling the radiative transfer in inhomogeneous three-dimensional media illuminated by a Gaussian beam is described. This approach, called the Spectral Spherical Harmonics Discrete Ordinate Method (SSHDOM), uses the Fourier expansion method to transform the three-dimensional radiative transfer into an one-dimensional equation in the spectral domain, and the Spherical Harmonics Discrete Ordinate Method (SHDOM) for its solution. Specifically, (i) the source function is represented in the spectral domain through a spherical harmonic expansion, (ii) the spectral one-dimensional radiative transfer equation is integrated along discrete ordinates through a spatial grid, and (iii) the solution method is based on the Picard iteration. Both SSHDOM and SHDOM algorithms are implemented in a common computer code.},
 bibtype = {article},
 author = {Doicu, Adrian and Mishchenko, Michael I. and Efremenko, Dmitry S. and Trautmann, Thomas},
 doi = {10.1016/j.jqsrt.2020.107386},
 journal = {Journal of Quantitative Spectroscopy and Radiative Transfer}
}

A new method for modeling the radiative transfer in inhomogeneous three-dimensional media illuminated by a Gaussian beam is described. This approach, called the Spectral Spherical Harmonics Discrete Ordinate Method (SSHDOM), uses the Fourier expansion method to transform the three-dimensional radiative transfer into an one-dimensional equation in the spectral domain, and the Spherical Harmonics Discrete Ordinate Method (SHDOM) for its solution. Specifically, (i) the source function is represented in the spectral domain through a spherical harmonic expansion, (ii) the spectral one-dimensional radiative transfer equation is integrated along discrete ordinates through a spatial grid, and (iii) the solution method is based on the Picard iteration. Both SSHDOM and SHDOM algorithms are implemented in a common computer code.

@article{
 title = {Spherical cap harmonic analysis (SCHA) for characterising the morphology of rough surface patches},
 type = {article},
 year = {2021},
 keywords = {Finite element,Fractal surfaces,Microstructures,Self-affine,Spherical cap harmonics,Surface parameterisation},
 pages = {837-856},
 volume = {393},
 websites = {https://doi.org/10.1016/j.powtec.2021.07.081},
 publisher = {The Author(s)},
 id = {5406b84e-4688-3a64-a94b-c228428f167f},
 created = {2023-05-03T13:16:39.062Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.295Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Shaqfa2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We use spherical cap harmonic (SCH) basis functions to analyse and reconstruct the morphology of scanned genus-0 rough surface patches with open edges. We first develop a novel one-to-one conformal mapping algorithm with minimal area distortion for parameterising a surface onto a polar spherical cap with a prescribed half angle. We then show that as a generalisation of the hemispherical harmonic analysis, the SCH analysis provides the most added value for small half angles, i.e., for nominally flat surfaces where the distortion introduced by the parameterisation algorithm is smaller when the surface is projected onto a spherical cap with a small half angle than onto a hemisphere. From the power spectral analysis of the expanded SCH coefficients, we estimate a direction-independent Hurst exponent. We also estimate the wavelengths associated with the orders of the SCH basis functions from the dimensions of the first degree ellipsoidal cap. By windowing the spectral domain, we limit the bandwidth of wavelengths included in a reconstructed surface geometry. This bandlimiting can be used for modifying surfaces, such as for generating finite or discrete element meshes for contact problems. The codes and data developed in this paper are made available under the GNU LGPLv2.1 license.},
 bibtype = {article},
 author = {Shaqfa, Mahmoud and Choi, Gary P.T. and Beyer, Katrin},
 doi = {10.1016/j.powtec.2021.07.081},
 journal = {Powder Technology}
}

We use spherical cap harmonic (SCH) basis functions to analyse and reconstruct the morphology of scanned genus-0 rough surface patches with open edges. We first develop a novel one-to-one conformal mapping algorithm with minimal area distortion for parameterising a surface onto a polar spherical cap with a prescribed half angle. We then show that as a generalisation of the hemispherical harmonic analysis, the SCH analysis provides the most added value for small half angles, i.e., for nominally flat surfaces where the distortion introduced by the parameterisation algorithm is smaller when the surface is projected onto a spherical cap with a small half angle than onto a hemisphere. From the power spectral analysis of the expanded SCH coefficients, we estimate a direction-independent Hurst exponent. We also estimate the wavelengths associated with the orders of the SCH basis functions from the dimensions of the first degree ellipsoidal cap. By windowing the spectral domain, we limit the bandwidth of wavelengths included in a reconstructed surface geometry. This bandlimiting can be used for modifying surfaces, such as for generating finite or discrete element meshes for contact problems. The codes and data developed in this paper are made available under the GNU LGPLv2.1 license.

@article{
 title = {Steerable 3D Spherical Neurons},
 type = {article},
 year = {2021},
 websites = {http://arxiv.org/abs/2106.13863},
 id = {2831707a-3154-3909-a085-0da74f4b8d5a},
 created = {2023-05-03T13:16:39.134Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:27.215Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Melnyk2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Emerging from low-level vision theory, steerable filters found their counterpart in prior work on steerable convolutional neural networks equivariant to rigid transformations. In our work, we propose a steerable feed-forward learning-based approach that consists of neurons with spherical decision surfaces and operates on point clouds. Such spherical neurons are obtained by conformal embedding of Euclidean space and have recently been revisited in the context of learning representations of point sets. Focusing on 3D geometry, we exploit the isometry property of spherical neurons and derive a 3D steerability constraint. After training spherical neurons to classify point clouds in a canonical orientation, we use a tetrahedron basis to quadruplicate the neurons and construct rotation-equivariant spherical filter banks. We then apply the derived constraint to interpolate the filter bank outputs and, thus, obtain a rotation-invariant network. Finally, we use a synthetic point set and real-world 3D skeleton data to verify our theoretical findings.},
 bibtype = {article},
 author = {Melnyk, Pavlo and Felsberg, Michael and Wadenbäck, Mårten}
}

Emerging from low-level vision theory, steerable filters found their counterpart in prior work on steerable convolutional neural networks equivariant to rigid transformations. In our work, we propose a steerable feed-forward learning-based approach that consists of neurons with spherical decision surfaces and operates on point clouds. Such spherical neurons are obtained by conformal embedding of Euclidean space and have recently been revisited in the context of learning representations of point sets. Focusing on 3D geometry, we exploit the isometry property of spherical neurons and derive a 3D steerability constraint. After training spherical neurons to classify point clouds in a canonical orientation, we use a tetrahedron basis to quadruplicate the neurons and construct rotation-equivariant spherical filter banks. We then apply the derived constraint to interpolate the filter bank outputs and, thus, obtain a rotation-invariant network. Finally, we use a synthetic point set and real-world 3D skeleton data to verify our theoretical findings.

@article{
 title = {Three-dimensional reconstruction of realistic stone-based materials with controllable stone inclusion geometries},
 type = {article},
 year = {2021},
 keywords = {DEM simulation,Overlapping detection,Realistic stone shape,Spherical Harmonic,Stochastic model,Stone inclusion,Stone-based material},
 pages = {124240},
 volume = {305},
 websites = {https://doi.org/10.1016/j.conbuildmat.2021.124240},
 publisher = {Elsevier Ltd},
 id = {f119f9c0-86f3-3135-a86e-eff7bc7576e3},
 created = {2023-05-03T13:16:39.255Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:14.533Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Wang2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Stone-based materials are heterogeneous construction materials that mainly consist of coarse stones and fine matrices. When using the numerical simulation tools (FEM, DEM, FDEM) to study the mechanical properties of the tone-based materials, the irregular stone-inclusion geometries cannot be neglected. The main objective of this study is to generate three-dimensional models of realistic stone-based materials with controllable stone-inclusion geometries. To achieve this goal, two major methodologies are adopted: (1) the Spherical Harmonic Transform (SHT) method and Inverse Monte-Carlo (IMC) algorithm are employed to randomly generate the 3D particle model of irregular stones with controllable geometries at three different shape scales, including form, roundness and roughness, and (2) based on the Spherical Harmonic function, an overlapping detection algorithm is proposed to facilitate the rapid allocation of the 3D SHT-based particles. The proposed algorithm can quantitatively control several geometrical features of the generated stone-based materials in an efficient and precise manner. Finally, the application of the proposed approach is demonstrated through the discrete modelling of stone-based materials with different stone contents and stone shapes. The proposed study has the significance to pave a viable pathway for stochastic modelling of stone-based materials pertaining to various construction and manufacture processes.},
 bibtype = {article},
 author = {Wang, Xiang and Yin, Zhen yu and Zhang, Jun qi and Xiong, Hao and Su, Dong},
 doi = {10.1016/j.conbuildmat.2021.124240},
 journal = {Construction and Building Materials},
 number = {January}
}

Stone-based materials are heterogeneous construction materials that mainly consist of coarse stones and fine matrices. When using the numerical simulation tools (FEM, DEM, FDEM) to study the mechanical properties of the tone-based materials, the irregular stone-inclusion geometries cannot be neglected. The main objective of this study is to generate three-dimensional models of realistic stone-based materials with controllable stone-inclusion geometries. To achieve this goal, two major methodologies are adopted: (1) the Spherical Harmonic Transform (SHT) method and Inverse Monte-Carlo (IMC) algorithm are employed to randomly generate the 3D particle model of irregular stones with controllable geometries at three different shape scales, including form, roundness and roughness, and (2) based on the Spherical Harmonic function, an overlapping detection algorithm is proposed to facilitate the rapid allocation of the 3D SHT-based particles. The proposed algorithm can quantitatively control several geometrical features of the generated stone-based materials in an efficient and precise manner. Finally, the application of the proposed approach is demonstrated through the discrete modelling of stone-based materials with different stone contents and stone shapes. The proposed study has the significance to pave a viable pathway for stochastic modelling of stone-based materials pertaining to various construction and manufacture processes.

@article{
 title = {Geometric Algebra Attention Networks for Small Point Clouds},
 type = {article},
 year = {2021},
 pages = {1-23},
 websites = {http://arxiv.org/abs/2110.02393},
 id = {08004e17-30c1-3c9a-b927-795ebc1c7caf},
 created = {2023-05-03T13:16:39.715Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.182Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Spellings2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Much of the success of deep learning is drawn from building architectures that properly respect underlying symmetry and structure in the data on which they operate - a set of considerations that have been united under the banner of geometric deep learning. Often problems in the physical sciences deal with relatively small sets of points in two- or three-dimensional space wherein translation, rotation, and permutation equivariance are important or even vital for models to be useful in practice. In this work, we present rotation- and permutation-equivariant architectures for deep learning on these small point clouds, composed of a set of products of terms from the geometric algebra and reductions over those products using an attention mechanism. The geometric algebra provides valuable mathematical structure by which to combine vector, scalar, and other types of geometric inputs in a systematic way to account for rotation invariance or covariance, while attention yields a powerful way to impose permutation equivariance. We demonstrate the usefulness of these architectures by training models to solve sample problems relevant to physics, chemistry, and biology.},
 bibtype = {article},
 author = {Spellings, Matthew}
}

Much of the success of deep learning is drawn from building architectures that properly respect underlying symmetry and structure in the data on which they operate - a set of considerations that have been united under the banner of geometric deep learning. Often problems in the physical sciences deal with relatively small sets of points in two- or three-dimensional space wherein translation, rotation, and permutation equivariance are important or even vital for models to be useful in practice. In this work, we present rotation- and permutation-equivariant architectures for deep learning on these small point clouds, composed of a set of products of terms from the geometric algebra and reductions over those products using an attention mechanism. The geometric algebra provides valuable mathematical structure by which to combine vector, scalar, and other types of geometric inputs in a systematic way to account for rotation invariance or covariance, while attention yields a powerful way to impose permutation equivariance. We demonstrate the usefulness of these architectures by training models to solve sample problems relevant to physics, chemistry, and biology.

@article{
 title = {Deep Hierarchical Rotation Invariance Learning with Exact Geometry Feature Representation for Point Cloud Classification},
 type = {article},
 year = {2021},
 pages = {9529-9535},
 volume = {2021-May},
 publisher = {IEEE},
 id = {e6064e13-9333-3ade-b010-4c6162a97cde},
 created = {2023-05-03T13:16:40.334Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-15T08:10:15.002Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Lin2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Rotation invariance is a crucial property for 3D object classification, which is still a challenging task. State-of-the-art deep learning-based works require a massive amount of data augmentation to tackle this problem. This is however inefficient and classification accuracy suffers a sharp drop in experiments with arbitrary rotations. We introduce a new descriptor that can globally and locally capture the surface geometry properties and is based on a combination of spherical harmonics energy and point feature representation. The proposed descriptor is proven to fulfill the rotation-invariant property. A limited bandwidth spherical harmonics energy descriptor globally describes a 3D shape and its rotation-invariant property is proven by utilizing the properties of a Wigner D-matrix, while the point feature representation captures the local features with a KNN to build the connection to its neighborhood. We propose a new network structure by extending PointNet++ with several adaptations that can hierarchically and efficiently exploit local rotation-invariant features. Extensive experimental results show that our proposed method dramatically outperforms most state-of-the-art approaches on standard rotation-augmented 3D object classification benchmarks as well as in robustness experiments on point perturbation, point density, and partial point clouds.},
 bibtype = {article},
 author = {Lin, Jianjie and Rickert, Markus and Knoll, Alois},
 doi = {10.1109/ICRA48506.2021.9561307},
 journal = {Proceedings - IEEE International Conference on Robotics and Automation},
 number = {Icra}
}

Rotation invariance is a crucial property for 3D object classification, which is still a challenging task. State-of-the-art deep learning-based works require a massive amount of data augmentation to tackle this problem. This is however inefficient and classification accuracy suffers a sharp drop in experiments with arbitrary rotations. We introduce a new descriptor that can globally and locally capture the surface geometry properties and is based on a combination of spherical harmonics energy and point feature representation. The proposed descriptor is proven to fulfill the rotation-invariant property. A limited bandwidth spherical harmonics energy descriptor globally describes a 3D shape and its rotation-invariant property is proven by utilizing the properties of a Wigner D-matrix, while the point feature representation captures the local features with a KNN to build the connection to its neighborhood. We propose a new network structure by extending PointNet++ with several adaptations that can hierarchically and efficiently exploit local rotation-invariant features. Extensive experimental results show that our proposed method dramatically outperforms most state-of-the-art approaches on standard rotation-augmented 3D object classification benchmarks as well as in robustness experiments on point perturbation, point density, and partial point clouds.

@article{
 title = {Spherical Multi-Modal Place Recognition for Heterogeneous Sensor Systems},
 type = {article},
 year = {2021},
 pages = {1743-1750},
 volume = {2021-May},
 publisher = {IEEE},
 id = {11c415c5-b520-3a4b-881d-88da37326322},
 created = {2023-05-03T13:16:40.652Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.409Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Bernreiter2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {In this paper, we propose a robust end-to-end multi-modal pipeline for place recognition where the sensor systems can differ from the map building to the query. Our approach operates directly on images and LiDAR scans without requiring any local feature extraction modules. By projecting the sensor data onto the unit sphere, we learn a multi-modal descriptor of partially overlapping scenes using a spherical convolutional neural network. The employed spherical projection model enables the support of arbitrary LiDAR and camera systems readily without losing information. Loop closure candidates are found using a nearest-neighbor lookup in the embedding space. We tackle the problem of correctly identifying the closest place by correlating the candidates' power spectra, obtaining a confidence value per prospect. Our estimate for the correct place corresponds then to the candidate with the highest confidence. We evaluate our proposal w.r.t. state-of-the-art approaches in place recognition using real-world data acquired using different sensors. Our approach can achieve a recall that is up to 10 % and 5 % higher than for a LiDAR- and vision-based system, respectively, when the sensor setup differs between model training and deployment. Additionally, our place selection can correctly identify up to 95 % matches from the candidate set.},
 bibtype = {article},
 author = {Bernreiter, Lukas and Ott, Lionel and Nieto, Juan and Siegwart, Roland and Cadena, Cesar},
 doi = {10.1109/ICRA48506.2021.9561078},
 journal = {Proceedings - IEEE International Conference on Robotics and Automation},
 number = {Icra}
}

In this paper, we propose a robust end-to-end multi-modal pipeline for place recognition where the sensor systems can differ from the map building to the query. Our approach operates directly on images and LiDAR scans without requiring any local feature extraction modules. By projecting the sensor data onto the unit sphere, we learn a multi-modal descriptor of partially overlapping scenes using a spherical convolutional neural network. The employed spherical projection model enables the support of arbitrary LiDAR and camera systems readily without losing information. Loop closure candidates are found using a nearest-neighbor lookup in the embedding space. We tackle the problem of correctly identifying the closest place by correlating the candidates' power spectra, obtaining a confidence value per prospect. Our estimate for the correct place corresponds then to the candidate with the highest confidence. We evaluate our proposal w.r.t. state-of-the-art approaches in place recognition using real-world data acquired using different sensors. Our approach can achieve a recall that is up to 10 % and 5 % higher than for a LiDAR- and vision-based system, respectively, when the sensor setup differs between model training and deployment. Additionally, our place selection can correctly identify up to 95 % matches from the candidate set.

@article{
 title = {FAST 3D ACOUSTIC SCATTERING VIA DISCRETE LAPLACIAN BASED IMPLICIT FUNCTION ENCODERS},
 type = {article},
 year = {2021},
 pages = {1-16},
 id = {6faadd4d-518e-3e3c-b7d3-8e90aea685fd},
 created = {2023-05-03T13:16:40.679Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:25.432Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Via2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 bibtype = {article},
 author = {Via, Cattering}
}

@article{
 title = {Spherical harmonics for shape-constrained 3d cell segmentation},
 type = {article},
 year = {2021},
 keywords = {3D Segmentation,Shape-Constrain,Spherical Harmonics},
 pages = {792-796},
 volume = {2021-April},
 id = {247366e8-6f7f-382b-a617-c4b25c1fbdbb},
 created = {2023-05-03T13:16:40.799Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.517Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Eschweiler2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {Recent microscopy imaging techniques allow to precisely analyze cell morphology in 3D image data. To process the vast amount of image data generated by current digitized imaging techniques, automated approaches are demanded more than ever. Segmentation approaches used for morphological analyses, however, are often prone to produce unnaturally shaped predictions, which in conclusion could lead to inaccurate experimental outcomes. In order to minimize further manual interaction, shape priors help to constrain the predictions to the set of natural variations. In this paper, we show how spherical harmonics can be used as an alternative way to inherently constrain the predictions of neural networks for the segmentation of cells in 3D microscopy image data. Benefits and limitations of the spherical harmonic representation are analyzed and final results are compared to other state-of-the-art approaches on two different data sets.},
 bibtype = {article},
 author = {Eschweiler, Dennis and Rethwisch, Malte and Koppers, Simon and Stegmaier, Johannes},
 doi = {10.1109/ISBI48211.2021.9433983},
 journal = {Proceedings - International Symposium on Biomedical Imaging},
 number = {3}
}

Recent microscopy imaging techniques allow to precisely analyze cell morphology in 3D image data. To process the vast amount of image data generated by current digitized imaging techniques, automated approaches are demanded more than ever. Segmentation approaches used for morphological analyses, however, are often prone to produce unnaturally shaped predictions, which in conclusion could lead to inaccurate experimental outcomes. In order to minimize further manual interaction, shape priors help to constrain the predictions to the set of natural variations. In this paper, we show how spherical harmonics can be used as an alternative way to inherently constrain the predictions of neural networks for the segmentation of cells in 3D microscopy image data. Benefits and limitations of the spherical harmonic representation are analyzed and final results are compared to other state-of-the-art approaches on two different data sets.

@article{
 title = {Point-based Acoustic Scattering for Interactive Sound Propagation via Surface Encoding},
 type = {article},
 year = {2021},
 pages = {909-915},
 id = {0af586ef-1e5b-33ec-8c01-5cd0392d292e},
 created = {2023-05-03T13:16:40.804Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.512Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Meng2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We present a novel geometric deep learning method to compute the acoustic scattering properties of geometric objects. Our learning algorithm uses a point cloud representation of objects to compute the scattering properties and integrates them with ray tracing for interactive sound propagation in dynamic scenes. We use discrete Laplacian-based surface encoders and approximate the neighborhood of each point using a shared multilayer perceptron. We show that our formulation is permutation invariant and present a neural network that computes the scattering function using spherical harmonics. Our approach can handle objects with arbitrary topologies and deforming models, and takes less than 1ms per object on a commodity GPU. We have analyzed the accuracy and perform validation on thousands of unseen 3D objects and highlight the benefits over other point-based geometric deep learning methods. To the best of our knowledge, this is the first real-time learning algorithm that can approximate the acoustic scattering properties of arbitrary objects with high accuracy.},
 bibtype = {article},
 author = {Meng, Hsien Yu and Tang, Zhenyu and Manocha, Dinesh},
 doi = {10.24963/ijcai.2021/126},
 journal = {IJCAI International Joint Conference on Artificial Intelligence}
}

We present a novel geometric deep learning method to compute the acoustic scattering properties of geometric objects. Our learning algorithm uses a point cloud representation of objects to compute the scattering properties and integrates them with ray tracing for interactive sound propagation in dynamic scenes. We use discrete Laplacian-based surface encoders and approximate the neighborhood of each point using a shared multilayer perceptron. We show that our formulation is permutation invariant and present a neural network that computes the scattering function using spherical harmonics. Our approach can handle objects with arbitrary topologies and deforming models, and takes less than 1ms per object on a commodity GPU. We have analyzed the accuracy and perform validation on thousands of unseen 3D objects and highlight the benefits over other point-based geometric deep learning methods. To the best of our knowledge, this is the first real-time learning algorithm that can approximate the acoustic scattering properties of arbitrary objects with high accuracy.

@article{
 title = {Learning acoustic scattering fields for dynamic interactive sound propagation},
 type = {article},
 year = {2021},
 keywords = {Computing methodologies-Computer graphics-Graphics},
 pages = {835-844},
 id = {9801dc07-6577-3746-996f-aa865918d79a},
 created = {2023-05-03T13:16:40.826Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-05-09T14:17:26.396Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Tang2021},
 folder_uuids = {4a65115c-c8d7-4bb4-831a-b057db051143},
 private_publication = {false},
 abstract = {We present a novel hybrid sound propagation algorithm for interactive applications. Our approach is designed for dynamic scenes and uses a neural network-based learned scattered field representation along with ray tracing to generate specular, diffuse, diffraction, and occlusion effects efficiently. We use geometric deep learning to approximate the acoustic scattering field using spherical harmonics. We use a large 3D dataset for training, and compare its accuracy with the ground truth generated using an accurate wave-based solver. The additional overhead of computing the learned scattered field at runtime is small and we demonstrate its interactive performance by generating plausible sound effects in dynamic scenes with diffraction and occlusion effects. We demonstrate the perceptual benefits of our approach based on an audio-visual user study.},
 bibtype = {article},
 author = {Tang, Zhenyu and Meng, Hsien Yu and Manocha, Dinesh},
 doi = {10.1109/VR50410.2021.00111},
 journal = {Proceedings - 2021 IEEE Conference on Virtual Reality and 3D User Interfaces, VR 2021}
}

We present a novel hybrid sound propagation algorithm for interactive applications. Our approach is designed for dynamic scenes and uses a neural network-based learned scattered field representation along with ray tracing to generate specular, diffuse, diffraction, and occlusion effects efficiently. We use geometric deep learning to approximate the acoustic scattering field using spherical harmonics. We use a large 3D dataset for training, and compare its accuracy with the ground truth generated using an accurate wave-based solver. The additional overhead of computing the learned scattered field at runtime is small and we demonstrate its interactive performance by generating plausible sound effects in dynamic scenes with diffraction and occlusion effects. We demonstrate the perceptual benefits of our approach based on an audio-visual user study.

@article{
 title = {Survey on the View Planning Problem for Reverse Engineering and Automated Control Applications},
 type = {article},
 year = {2021},
 keywords = {3D reconstruction,Measurement,Object inspection,Optical sensor,Point cloud,View planning},
 pages = {103094},
 volume = {141},
 websites = {https://doi.org/10.1016/j.cad.2021.103094},
 publisher = {Elsevier Ltd},
 id = {28063d9d-98d5-335d-83c6-4d2c5ffa9491},
 created = {2023-07-07T07:53:43.914Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-08-08T11:39:19.266Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {e4a1ea0f-69ae-4053-94cc-503201fc6c67},
 private_publication = {false},
 abstract = {At present, optical sensors are being widely used to realize high quality control or reverse engineering of products, systems, buildings, environments or human bodies. Although the intrinsic characteristics of such breakthrough technologies may vary, ensuring complete acquisition relies on the definition of the optimal acquisition planning. To this end, the view planning problem (VPP) must be solved to automatically determine the optimal positions and/or trajectories of the acquisition devices to fully cover the part to be digitized. Such an automatization of the entire acquisition process is of considerably interest in the context of Industry 4.0. The aim of this paper is to review the state of the art works addressing the view planning problem and to identify the future challenges and possible research directions. First, the paper introduces a set of criteria to analyze the available methods, grouped into several macrocategories. The categories are presented and formalized to clearly understand the backbone and similarities of the grouped methods. Second, the paper describes and characterizes the available methods, based on their analysis according to the adopted criteria. The results of this extensive analysis clearly highlight the open issues and future challenges.},
 bibtype = {article},
 author = {Peuzin-Jubert, Manon and Polette, Arnaud and Nozais, Dominique and Mari, Jean Luc and Pernot, Jean Philippe},
 doi = {10.1016/j.cad.2021.103094},
 journal = {CAD Computer Aided Design}
}

At present, optical sensors are being widely used to realize high quality control or reverse engineering of products, systems, buildings, environments or human bodies. Although the intrinsic characteristics of such breakthrough technologies may vary, ensuring complete acquisition relies on the definition of the optimal acquisition planning. To this end, the view planning problem (VPP) must be solved to automatically determine the optimal positions and/or trajectories of the acquisition devices to fully cover the part to be digitized. Such an automatization of the entire acquisition process is of considerably interest in the context of Industry 4.0. The aim of this paper is to review the state of the art works addressing the view planning problem and to identify the future challenges and possible research directions. First, the paper introduces a set of criteria to analyze the available methods, grouped into several macrocategories. The categories are presented and formalized to clearly understand the backbone and similarities of the grouped methods. Second, the paper describes and characterizes the available methods, based on their analysis according to the adopted criteria. The results of this extensive analysis clearly highlight the open issues and future challenges.

@article{
 title = {Low Power Processors and Image Sensors for Vision-Based IoT Devices: A Review},
 type = {article},
 year = {2021},
 keywords = {Internet of Things,image sensor,low power,machine vision,processor},
 pages = {1172-1186},
 volume = {21},
 month = {1},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {15},
 id = {372104ec-fb2d-381f-987a-e34f771ab0ae},
 created = {2023-11-07T09:49:41.472Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:40:49.311Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {31 cites},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {With the advancements of the Internet of Things (IoT) technology, applications of battery powered machine vision based IoT devices is rapidly growing. While numerous research works are being conducted to develop low power hardware solutions for IoT devices, image capture and image processing remain high power demanding processes leading to a short battery life. However, the power consumption of the machine vision based IoT devices can be minimized by the careful optimization of the hardware components that are used is these devices. In this article, we present a review of low power machine vision hardware components for the IoT applications. A guide to selecting the optimum processors and image sensors for a given battery powered machine vision based IoT device is presented. Next, the factors that must be considered when selecting processors and image sensors for a given IoT application are discussed, and selection criteria for the processors and image sensors are established. Then, the current commercially available hardware components are reviewed in accordance with the established selection criteria. Finally, the research trends in the field of battery powered machine vision based IoT devices are discussed, and the potential future research directions in the field are presented.},
 bibtype = {article},
 author = {Maheepala, Malith and Joordens, Matthew A. and Kouzani, Abbas Z.},
 doi = {10.1109/JSEN.2020.3015932},
 journal = {IEEE Sensors Journal},
 number = {2}
}

With the advancements of the Internet of Things (IoT) technology, applications of battery powered machine vision based IoT devices is rapidly growing. While numerous research works are being conducted to develop low power hardware solutions for IoT devices, image capture and image processing remain high power demanding processes leading to a short battery life. However, the power consumption of the machine vision based IoT devices can be minimized by the careful optimization of the hardware components that are used is these devices. In this article, we present a review of low power machine vision hardware components for the IoT applications. A guide to selecting the optimum processors and image sensors for a given battery powered machine vision based IoT device is presented. Next, the factors that must be considered when selecting processors and image sensors for a given IoT application are discussed, and selection criteria for the processors and image sensors are established. Then, the current commercially available hardware components are reviewed in accordance with the established selection criteria. Finally, the research trends in the field of battery powered machine vision based IoT devices are discussed, and the potential future research directions in the field are presented.

@article{
 title = {Embedded Intelligence on FPGA: Survey, Applications and Challenges},
 type = {article},
 year = {2021},
 keywords = {FPGA,artificial intelligence,deep learning,embedded intelligence,embedded systems,neural networks},
 pages = {895},
 volume = {10},
 websites = {https://doi.org/10.3390/electronics},
 month = {4},
 day = {8},
 id = {e554b3d6-6193-346e-9b35-d72b40f1b16c},
 created = {2023-11-07T09:56:35.794Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:40:29.918Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {85 cites},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Embedded intelligence (EI) is an emerging research field and has the objective to incorporate machine learning algorithms and intelligent decision-making capabilities into mobile and embedded devices or systems. There are several challenges to be addressed to realize efficient EI implementations in hardware such as the need for: (1) high computational processing; (2) low power consumption (or high energy efficiency); and (3) scalability to accommodate different network sizes and topologies. In recent years, an emerging hardware technology which has demonstrated strong potential and capabilities for EI implementations is the FPGA (field programmable gate array) technology. This paper presents an overview and review of embedded intelligence on FPGA with a focus on applications, platforms and challenges. There are four main classification and thematic descriptors which are reviewed and discussed in this paper for EI: (1) EI techniques including machine learning and neural networks, deep learning, expert systems, fuzzy intelligence, swarm intelligence, self-organizing map (SOM) and extreme learning; (2) applications for EI including object detection and recognition, indoor localization and surveillance monitoring, and other EI applications; (3) hardware and platforms for EI; and (4) challenges for EI. The paper aims to introduce interested researchers to this area and motivate the development of practical FPGA solutions for EI deployment.},
 bibtype = {article},
 author = {Seng, K P and Lee, P J and Ang, L M},
 doi = {10.3390/electronics},
 journal = {Electronics}
}

Embedded intelligence (EI) is an emerging research field and has the objective to incorporate machine learning algorithms and intelligent decision-making capabilities into mobile and embedded devices or systems. There are several challenges to be addressed to realize efficient EI implementations in hardware such as the need for: (1) high computational processing; (2) low power consumption (or high energy efficiency); and (3) scalability to accommodate different network sizes and topologies. In recent years, an emerging hardware technology which has demonstrated strong potential and capabilities for EI implementations is the FPGA (field programmable gate array) technology. This paper presents an overview and review of embedded intelligence on FPGA with a focus on applications, platforms and challenges. There are four main classification and thematic descriptors which are reviewed and discussed in this paper for EI: (1) EI techniques including machine learning and neural networks, deep learning, expert systems, fuzzy intelligence, swarm intelligence, self-organizing map (SOM) and extreme learning; (2) applications for EI including object detection and recognition, indoor localization and surveillance monitoring, and other EI applications; (3) hardware and platforms for EI; and (4) challenges for EI. The paper aims to introduce interested researchers to this area and motivate the development of practical FPGA solutions for EI deployment.

@article{
 title = {Revisiting Point Cloud Shape Classification with a Simple and Effective Baseline},
 type = {article},
 year = {2021},
 websites = {https://github.com/},
 id = {591937b5-b69d-3fa4-8e4a-52e70623987f},
 created = {2024-08-29T12:58:09.643Z},
 accessed = {2024-08-29},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2024-08-29T12:58:38.078Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {8b2deb8c-cbcd-4d63-ade5-3dac1fc50168},
 private_publication = {false},
 abstract = {Processing point cloud data is an important component of many real-world systems. As such, a wide variety of point-based approaches have been proposed, reporting steady benchmark improvements over time. We study the key ingredients of this progress and uncover two critical results. First, we find that auxiliary factors like different evaluation schemes, data augmentation strategies , and loss functions, which are independent of the model architecture, make a large difference in performance. The differences are large enough that they obscure the effect of architecture. When these factors are controlled for, Point-Net++, a relatively older network, performs competitively with recent methods. Second, a very simple projection-based method, which we refer to as SimpleView, performs surprisingly well. It achieves on par or better results than sophisticated state-of-the-art methods on ModelNet40 while being half the size of PointNet++. It also outperforms state-of-the-art methods on ScanOb-jectNN, a real-world point cloud benchmark, and demonstrates better cross-dataset generalization. Code is available at https://github.com/ princeton-vl/SimpleView.},
 bibtype = {article},
 author = {Goyal, Ankit and Law, Hei and Liu, Bowei and Newell, Alejandro and Deng, Jia}
}

Processing point cloud data is an important component of many real-world systems. As such, a wide variety of point-based approaches have been proposed, reporting steady benchmark improvements over time. We study the key ingredients of this progress and uncover two critical results. First, we find that auxiliary factors like different evaluation schemes, data augmentation strategies , and loss functions, which are independent of the model architecture, make a large difference in performance. The differences are large enough that they obscure the effect of architecture. When these factors are controlled for, Point-Net++, a relatively older network, performs competitively with recent methods. Second, a very simple projection-based method, which we refer to as SimpleView, performs surprisingly well. It achieves on par or better results than sophisticated state-of-the-art methods on ModelNet40 while being half the size of PointNet++. It also outperforms state-of-the-art methods on ScanOb-jectNN, a real-world point cloud benchmark, and demonstrates better cross-dataset generalization. Code is available at https://github.com/ princeton-vl/SimpleView.

@article{
 title = {High-Utilization, High-Flexibility Depth-First CNN Coprocessor for Image Pixel Processing on FPGA},
 type = {article},
 year = {2021},
 keywords = {Convolutional neural network (CNN),depth-first (DF),field-programmable gate array (FPGA),flexible processor design,high throughput,pixel processing},
 pages = {461-471},
 volume = {29},
 month = {3},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {6d63e917-93ac-3816-8eb3-33006e453f24},
 created = {2025-01-20T12:09:19.479Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:24:37.532Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {33 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Recently, CNNs are increasingly exploited for pixel processing tasks, such as denoising, which opens up new challenges due to the increased activation and operation count. This article presents a CNN coprocessor architecture to solve these challenges on field-programmable gate array (FPGA) through four main contributions. First, the I/O communication between the host processor and the FPGA is reduced to a minimum using a depth-first (DF) principle. Three new DF approaches are presented. Second, to ensure high throughput, the increased parallelization opportunities of the proposed line-based DF operation are analyzed. Third, introducing programmability to the compute array is introduced to enable a broad deployment while maintaining high utilization of the available multipliers digital signal processings (DSPs), independently of the kernel dimensions and without control of the host processor. This is in contrast with many state-of-the-art FPGA implementations, focusing on only one algorithm and/or one kernel topology. Fourth, a model is built to investigate the influence of architecture parameters and show the benefits of DF. The scalable design can be deployed on a wide range of FPGAs, maintaining 78%-93% DSP utilization across all algorithms (denoising, optical flow, depth estimation, segmentation, and super-resolution) and FPGA platforms. Up to 695 GOPS is achieved on a Zynq XCZU9EG board, matching state-of-the-art performance with a more flexible design. The throughput is compared with other pixel processing architectures on FPGA.},
 bibtype = {article},
 author = {Colleman, Steven and Verhelst, Marian},
 doi = {10.1109/TVLSI.2020.3046125},
 journal = {IEEE Transactions on Very Large Scale Integration (VLSI) Systems},
 number = {3}
}

Recently, CNNs are increasingly exploited for pixel processing tasks, such as denoising, which opens up new challenges due to the increased activation and operation count. This article presents a CNN coprocessor architecture to solve these challenges on field-programmable gate array (FPGA) through four main contributions. First, the I/O communication between the host processor and the FPGA is reduced to a minimum using a depth-first (DF) principle. Three new DF approaches are presented. Second, to ensure high throughput, the increased parallelization opportunities of the proposed line-based DF operation are analyzed. Third, introducing programmability to the compute array is introduced to enable a broad deployment while maintaining high utilization of the available multipliers digital signal processings (DSPs), independently of the kernel dimensions and without control of the host processor. This is in contrast with many state-of-the-art FPGA implementations, focusing on only one algorithm and/or one kernel topology. Fourth, a model is built to investigate the influence of architecture parameters and show the benefits of DF. The scalable design can be deployed on a wide range of FPGAs, maintaining 78%-93% DSP utilization across all algorithms (denoising, optical flow, depth estimation, segmentation, and super-resolution) and FPGA platforms. Up to 695 GOPS is achieved on a Zynq XCZU9EG board, matching state-of-the-art performance with a more flexible design. The throughput is compared with other pixel processing architectures on FPGA.

@article{
 title = {FPGA implementation for CNN-based optical remote sensing object detection},
 type = {article},
 year = {2021},
 keywords = {CNN,Deep learning,FPGA,Hardware implementation,Object detection,Remote sensing,You-only-look-once (YOLO)},
 pages = {1-24},
 volume = {10},
 month = {2},
 publisher = {MDPI AG},
 day = {1},
 id = {231ff9c7-1f91-36d9-bbf3-4c94a6ea436a},
 created = {2025-04-07T08:23:01.845Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:22:30.323Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {75 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {In recent years, convolutional neural network (CNN)-based methods have been widely used for optical remote sensing object detection and have shown excellent performance. Some aerospace systems, such as satellites or aircrafts, need to adopt these methods to observe objects on the ground. Due to the limited budget of the logical resources and power consumption in these systems, an embedded device is a good choice to implement the CNN-based methods. However, it is still a challenge to strike a balance between performance and power consumption. In this paper, we propose an efficient hardware-implementation method for optical remote sensing object detection. Firstly, we optimize the CNN-based model for hardware implementation, which establishes a foundation for efficiently mapping the network on a field-programmable gate array (FPGA). In addition, we propose a hardware architecture for the CNN-based remote sensing object detection model. In this architecture, a general processing engine (PE) is proposed to implement multiple types of convolutions in the network using the uniform module. An efficient data storage and access scheme is also proposed, and it achieves low-latency calculations and a high memory bandwidth utilization rate. Finally, we deployed the improved YOLOv2 network on a Xilinx ZYNQ xc7z035 FPGA to evaluate the performance of our design. The experimental results show that the performance of our implementation on an FPGA is only 0.18% lower than that on a graphics processing unit (GPU) in mean average precision (mAP). Under a 200 MHz working frequency, our design achieves a throughput of 111.5 giga-operations per second (GOP/s) with a 5.96 W on-chip power consumption. Comparison with the related works demonstrates that the proposed design has obvious advantages in terms of energy efficiency and that it is suitable for deployment on embedded devices.},
 bibtype = {article},
 author = {Zhang, Ning and Wei, Xin and Chen, He and Liu, Wenchao},
 doi = {10.3390/electronics10030282},
 journal = {Electronics (Switzerland)},
 number = {3}
}

In recent years, convolutional neural network (CNN)-based methods have been widely used for optical remote sensing object detection and have shown excellent performance. Some aerospace systems, such as satellites or aircrafts, need to adopt these methods to observe objects on the ground. Due to the limited budget of the logical resources and power consumption in these systems, an embedded device is a good choice to implement the CNN-based methods. However, it is still a challenge to strike a balance between performance and power consumption. In this paper, we propose an efficient hardware-implementation method for optical remote sensing object detection. Firstly, we optimize the CNN-based model for hardware implementation, which establishes a foundation for efficiently mapping the network on a field-programmable gate array (FPGA). In addition, we propose a hardware architecture for the CNN-based remote sensing object detection model. In this architecture, a general processing engine (PE) is proposed to implement multiple types of convolutions in the network using the uniform module. An efficient data storage and access scheme is also proposed, and it achieves low-latency calculations and a high memory bandwidth utilization rate. Finally, we deployed the improved YOLOv2 network on a Xilinx ZYNQ xc7z035 FPGA to evaluate the performance of our design. The experimental results show that the performance of our implementation on an FPGA is only 0.18% lower than that on a graphics processing unit (GPU) in mean average precision (mAP). Under a 200 MHz working frequency, our design achieves a throughput of 111.5 giga-operations per second (GOP/s) with a 5.96 W on-chip power consumption. Comparison with the related works demonstrates that the proposed design has obvious advantages in terms of energy efficiency and that it is suitable for deployment on embedded devices.

@article{
 title = {CNN-on-AWS: Efficient Allocation of Multikernel Applications on Multi-FPGA Platforms},
 type = {article},
 year = {2021},
 keywords = {Allocation,convolutional neural networks (CNNs),multi-FPGA,optimization},
 pages = {301-314},
 volume = {40},
 month = {2},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {98dce73e-53aa-3794-90fb-d64eebae14b2},
 created = {2025-04-07T08:23:02.904Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:21:43.421Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {26 cites},
 folder_uuids = {e1046472-c458-4ece-b1ec-2fb78d2d5eb6,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {Multi-FPGA platforms, like Amazon AWS F1, can run in the cloud multikernel pipelined applications, like convolutional neural networks (CNNs), with excellent performance and lower energy consumption than CPUs or GPUs. We propose a method to efficiently map these applications on multi-FPGA platforms to maximize the application throughput. Our methodology finds, for the given resources, the optimal number of parallel instances of each kernel in the pipeline and their allocation to one or more among the available FPGAs. We obtain this by formulating and solving a mixed-integer, nonlinear optimization problem, in which we model the performance of each component and the duration of the phases in which the accelerated computation can be split into, namely: 1) data transfer from a host CPU to the DDR memory of each FPGA; 2) data transfer from FPGA DDR to FPGA on-chip memory; 3) kernel computation on the FPGA; 4) data transfer from FPGA on-chip memory to FPGA DDR; and 5) data transfer from FPGA DDR to host. Finding the optimal solution using a mixed-integer nonlinear programming (MINLP) solver is often highly inefficient. Hence, we provide a fast heuristic method that according to our experiments can be much more efficient than the MINLP solver and finds comparable results. For larger problems (more CNN layers), our heuristic method can quickly find (several thousand times faster) much better solutions than the MINLP solver, even if we run the latter for a very long time.},
 bibtype = {article},
 author = {Shan, Junnan and Lazarescu, Mihai T. and Cortadella, Jordi and Lavagno, Luciano and Casu, Mario R.},
 doi = {10.1109/TCAD.2020.2994256},
 journal = {IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems},
 number = {2}
}

Multi-FPGA platforms, like Amazon AWS F1, can run in the cloud multikernel pipelined applications, like convolutional neural networks (CNNs), with excellent performance and lower energy consumption than CPUs or GPUs. We propose a method to efficiently map these applications on multi-FPGA platforms to maximize the application throughput. Our methodology finds, for the given resources, the optimal number of parallel instances of each kernel in the pipeline and their allocation to one or more among the available FPGAs. We obtain this by formulating and solving a mixed-integer, nonlinear optimization problem, in which we model the performance of each component and the duration of the phases in which the accelerated computation can be split into, namely: 1) data transfer from a host CPU to the DDR memory of each FPGA; 2) data transfer from FPGA DDR to FPGA on-chip memory; 3) kernel computation on the FPGA; 4) data transfer from FPGA on-chip memory to FPGA DDR; and 5) data transfer from FPGA DDR to host. Finding the optimal solution using a mixed-integer nonlinear programming (MINLP) solver is often highly inefficient. Hence, we provide a fast heuristic method that according to our experiments can be much more efficient than the MINLP solver and finds comparable results. For larger problems (more CNN layers), our heuristic method can quickly find (several thousand times faster) much better solutions than the MINLP solver, even if we run the latter for a very long time.

@inproceedings{
 title = {FPGA Implementation of Object Detection Accelerator Based on Vitis-AI},
 type = {inproceedings},
 year = {2021},
 keywords = {FPGA,Object Detection,Parallel Processing,Vitis AI,YOLOv3},
 pages = {571-577},
 month = {5},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {21},
 id = {8a4b38b1-946f-3a4a-aa95-9fef26bad7b3},
 created = {2025-04-07T10:45:52.927Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:37:48.258Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {62 cites<br/><br/><b>Paper description</b><br/>FPGA Implementation of Object Detection Accelerator Based on<br/>Vitis-AI. <br/>YOLOv3 for object detection on FPGA to achieve high throughput and efficiency.<br/><br/><b>Methods</b>: YOLOv3 with Vitis AI optimizations like INT8 quantization, pruning, and pipelining.<br/><br/><b>Tools</b>: Xilinx ZCU104, Vitis AI, PASCAL VOC dataset.<br/><br/><b>Materials</b>: Trained YOLOv3 models (e.g., yolov3 voc, yolov3 bdd).<br/><br/><b>Results</b><br/>- Single-thread FPS: 84.5, Multithread FPS: 206.7.<br/>- Energy efficiency improved 13x compared to GPU.<br/>- FPGA achieved 2.4x better throughput than GPU while consuming only 25W vs. 126W.<br/>- Quantization with high accuracy},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {The emergence of YOLOv3 makes it possible to detect small targets. Due to the characteristics of the YOLO network itself, the YOLOv3 network has exceptionally high requirements for computing power and memory bandwidth and it usually needs to be deployed on a dedicated hardware acceleration platform. FPGAs is a logically reconfigurable hardware chip with substantial advantages in terms of performance and power consumption, so it is a good choice to deploy a deep convolutional network. In the research of this paper, we proposed a reconfigurable YOLOv3 FPGA hardware accelerator based on the AXI bus ARM+FPGA architecture. The YOLOv3 network quantifies through Vitis AI, and a series of operations such as model compression and data pre-processing can save accelerator chips and the access time of external storage. Pipeline operation enables FPGAs to achieve higher throughput. Compared with the GPU implementation of the YOLOv3 model, it is found that the hardware implementation of the FPGA-based YOLOv3 accelerator has lower energy consumption and can achieve higher throughput.},
 bibtype = {inproceedings},
 author = {Wang, Jin and Gu, Shenshen},
 doi = {10.1109/ICIST52614.2021.9440554},
 booktitle = {2021 11th International Conference on Information Science and Technology, ICIST 2021}
}

The emergence of YOLOv3 makes it possible to detect small targets. Due to the characteristics of the YOLO network itself, the YOLOv3 network has exceptionally high requirements for computing power and memory bandwidth and it usually needs to be deployed on a dedicated hardware acceleration platform. FPGAs is a logically reconfigurable hardware chip with substantial advantages in terms of performance and power consumption, so it is a good choice to deploy a deep convolutional network. In the research of this paper, we proposed a reconfigurable YOLOv3 FPGA hardware accelerator based on the AXI bus ARM+FPGA architecture. The YOLOv3 network quantifies through Vitis AI, and a series of operations such as model compression and data pre-processing can save accelerator chips and the access time of external storage. Pipeline operation enables FPGAs to achieve higher throughput. Compared with the GPU implementation of the YOLOv3 model, it is found that the hardware implementation of the FPGA-based YOLOv3 accelerator has lower energy consumption and can achieve higher throughput.

@article{
 title = {Fast convolutional neural networks on FPGAs with hls4ml},
 type = {article},
 year = {2021},
 volume = {2},
 websites = {http://arxiv.org/abs/2101.05108,http://dx.doi.org/10.1088/2632-2153/ac0ea1},
 month = {1},
 day = {13},
 id = {1167efe0-6e68-3d3c-82c8-149fc0776cbd},
 created = {2025-04-07T13:57:57.438Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:36:57.731Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {161 cites<br/><br/><b>hls4ml </b>Original paper<br/>Explained and exemplified},
 folder_uuids = {236cce89-f2ce-4369-93c5-ade3edd69a79,1bffc8fa-4e6e-48c8-b694-323af7fbd0e8},
 private_publication = {false},
 abstract = {We introduce an automated tool for deploying ultra low-latency, low-power deep neural networks with convolutional layers on FPGAs. By extending the hls4ml library, we demonstrate an inference latency of $5\,\mu$s using convolutional architectures, targeting microsecond latency applications like those at the CERN Large Hadron Collider. Considering benchmark models trained on the Street View House Numbers Dataset, we demonstrate various methods for model compression in order to fit the computational constraints of a typical FPGA device used in trigger and data acquisition systems of particle detectors. In particular, we discuss pruning and quantization-aware training, and demonstrate how resource utilization can be significantly reduced with little to no loss in model accuracy. We show that the FPGA critical resource consumption can be reduced by 97% with zero loss in model accuracy, and by 99% when tolerating a 6% accuracy degradation.},
 bibtype = {article},
 author = {Aarrestad, Thea and Loncar, Vladimir and Ghielmetti, Nicolò and Pierini, Maurizio and Summers, Sioni and Ngadiuba, Jennifer and Petersson, Christoffer and Linander, Hampus and Iiyama, Yutaro and Di Guglielmo, Giuseppe and Duarte, Javier and Harris, Philip and Rankin, Dylan and Jindariani, Sergo and Pedro, Kevin and Tran, Nhan and Liu, Mia and Kreinar, Edward and Wu, Zhenbin and Hoang, Duc},
 doi = {10.1088/2632-2153/ac0ea1},
 journal = {Machine Learning: Science and Technology},
 number = {4}
}

We introduce an automated tool for deploying ultra low-latency, low-power deep neural networks with convolutional layers on FPGAs. By extending the hls4ml library, we demonstrate an inference latency of $5\,\mu$s using convolutional architectures, targeting microsecond latency applications like those at the CERN Large Hadron Collider. Considering benchmark models trained on the Street View House Numbers Dataset, we demonstrate various methods for model compression in order to fit the computational constraints of a typical FPGA device used in trigger and data acquisition systems of particle detectors. In particular, we discuss pruning and quantization-aware training, and demonstrate how resource utilization can be significantly reduced with little to no loss in model accuracy. We show that the FPGA critical resource consumption can be reduced by 97% with zero loss in model accuracy, and by 99% when tolerating a 6% accuracy degradation.

@techreport{
 title = {Why is FPGA-GPU Heterogeneity the Best Option for Embedded Deep Neural Networks?},
 type = {techreport},
 year = {2021},
 websites = {https://doi.org/10.48550/arXiv.2102.01343},
 month = {2},
 day = {5},
 institution = {Workshop on System-level Design Methods for Deep Learning on Heterogeneous Architectures (SLOHA 2021)},
 id = {15c0eb5d-22ac-34be-ae69-1cf3967b6993},
 created = {2025-04-11T06:58:37.465Z},
 accessed = {2025-01-19},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-04T14:41:26.773Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {16 cites  <br/><br/>Abtract:   <br/><b>Direct Hardware Mapping for CNN on FPGAs</b>  <br/><br/>Advantage of FPGA-GPU hydrid: <br/><b>outperfroms GPU around 20% better energy comption and latency.</b>   <br/><br/>The DHM: fixed-point computation approach <b>(8-bit)</b><br/>Hardware architecture specificially important for enbedded systems: <br/>- show viable alternative of FPGA for small layers DL accelarators <br/>- compare   FPGA with enbedded GPU and show effiency of hybrid approach  <br/><br/>The DHM requires high usage of resources in FPGAs, only used for small designs, might be usefull to use GPU for reduce memory accesses.  <br/><br/>FPGAs limit the depth of convolution filters that can be directly mapped. Shows table for images: 224x224x3, FGPA outperforms GPU significantly for small number of of filters and sizes.    <br/><br/>Method and Materials:    <br/>Nvidia <b>Jetson TX2</b> CPU-GPU board Intel <b>Cyclone10GX FPGA</b><br/>CNNs: SqueezeNet, MobileNetv2, and ShuffleNetv2  ImageNet pre-trained mobile CNN models were obtained from <b>Pytorch</b> <br/><b>Power Estimation tool </b>from Intel Quartus Pro Edition},
 folder_uuids = {8f28bc6b-cc43-4e5b-b5ff-828fcf8c1cbc},
 private_publication = {false},
 abstract = {Graphics Processing Units (GPUs) are currently the dominating programmable architecture for Deep Learning (DL) accelerators. The adoption of Field Programmable Gate Arrays (FPGAs) in DL accelerators is however getting momentum. In this paper, we demonstrate that Direct Hardware Mapping (DHM) of a Convolutional Neural Network (CNN) on an embedded FPGA substantially outperforms a GPU implementation in terms of energy efficiency and execution time. However, DHM is highly resource intensive and cannot fully substitute the GPU when implementing a state-of-the-art CNN. We thus propose a hybrid FPGA-GPU DL acceleration method and demonstrate that heterogeneous acceleration outperforms GPU acceleration even including communication overheads.
Experimental results are conducted on a heterogeneous multi-platform setup embedding an Nvidia(R) Jetson TX2 CPU-GPU board and an Intel(R) Cyclone10GX FPGA board. The SqueezeNet, MobileNetv2, and ShuffleNetv2 mobile-oriented CNNs are experimented. We show that heterogeneous FPG-AGPU acceleration outperforms GPU acceleration for classification inference task over MobileNetv2 (12%-30% energy reduction, 4% to 26% latency reduction), SqueezeNet (21%-28% energy reduction, same latency), and ShuffleNetv2 (25% energy reduction, 21% latency reduction).},
 bibtype = {techreport},
 author = {Carballo-Hernández, Walther and Pelcat, Maxime and Berry, François},
 doi = {https://doi.org/10.48550/arXiv.2102.01343}
}

Graphics Processing Units (GPUs) are currently the dominating programmable architecture for Deep Learning (DL) accelerators. The adoption of Field Programmable Gate Arrays (FPGAs) in DL accelerators is however getting momentum. In this paper, we demonstrate that Direct Hardware Mapping (DHM) of a Convolutional Neural Network (CNN) on an embedded FPGA substantially outperforms a GPU implementation in terms of energy efficiency and execution time. However, DHM is highly resource intensive and cannot fully substitute the GPU when implementing a state-of-the-art CNN. We thus propose a hybrid FPGA-GPU DL acceleration method and demonstrate that heterogeneous acceleration outperforms GPU acceleration even including communication overheads. Experimental results are conducted on a heterogeneous multi-platform setup embedding an Nvidia(R) Jetson TX2 CPU-GPU board and an Intel(R) Cyclone10GX FPGA board. The SqueezeNet, MobileNetv2, and ShuffleNetv2 mobile-oriented CNNs are experimented. We show that heterogeneous FPG-AGPU acceleration outperforms GPU acceleration for classification inference task over MobileNetv2 (12%-30% energy reduction, 4% to 26% latency reduction), SqueezeNet (21%-28% energy reduction, same latency), and ShuffleNetv2 (25% energy reduction, 21% latency reduction).

@article{
 title = {An Optimized FPGA-Based Real-Time NDT for 3D-LiDAR Localization in Smart Vehicles},
 type = {article},
 year = {2021},
 keywords = {3D LiDAR,3D point cloud,FPGA,Smart vehicles,energy-efficient,localization,real-time},
 pages = {3167-3171},
 volume = {68},
 month = {9},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 day = {1},
 id = {0d1e719f-0d48-39f3-b1e4-1a2a68f5aa4a},
 created = {2025-06-03T16:14:32.860Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:34:27.196Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {22 cites},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Light detection and ranging (LiDAR) based normal distribution transform (NDT) is a popular localization algorithm in smart vehicles for its robustness and accuracy. To meet the requirement of higher speed of smart vehicles, LiDAR point clouds become increasingly large due to the increase of not only the number of laser channels but also their frame rates. However, previous NDT implementations fail to achieve real-time performance when handling large point clouds because they require an enormous number of search operations and perform intensive computations per search operation. To address this issue, we propose an optimized NDT algorithm and its FPGA implementation. First, we effectively speed up each search operation by proposing a new non-recursive and memory-efficient data structure, called occupation-aware-voxel-structure (OAVS). Second, we significantly reduce the number of search operations by removing redundant searches with an optimized semantic-assisted OAVS-based NDT algorithm (SEO-NDT). Third, we further improve the real-time and energy efficiency by proposing a streaming FPGA accelerator architecture for SEO-NDT. Compared with the state-of-the-art embedded CPU and GPU implementations, our FPGA implementation achieves up to 35.85x and 2.44x execution time speed up, respectively.},
 bibtype = {article},
 author = {Deng, Qi and Sun, Hao and Chen, Fupeng and Shu, Yuhao and Wang, Hui and Ha, Yajun},
 doi = {10.1109/TCSII.2021.3095764},
 journal = {IEEE Transactions on Circuits and Systems II: Express Briefs},
 number = {9}
}

Light detection and ranging (LiDAR) based normal distribution transform (NDT) is a popular localization algorithm in smart vehicles for its robustness and accuracy. To meet the requirement of higher speed of smart vehicles, LiDAR point clouds become increasingly large due to the increase of not only the number of laser channels but also their frame rates. However, previous NDT implementations fail to achieve real-time performance when handling large point clouds because they require an enormous number of search operations and perform intensive computations per search operation. To address this issue, we propose an optimized NDT algorithm and its FPGA implementation. First, we effectively speed up each search operation by proposing a new non-recursive and memory-efficient data structure, called occupation-aware-voxel-structure (OAVS). Second, we significantly reduce the number of search operations by removing redundant searches with an optimized semantic-assisted OAVS-based NDT algorithm (SEO-NDT). Third, we further improve the real-time and energy efficiency by proposing a streaming FPGA accelerator architecture for SEO-NDT. Compared with the state-of-the-art embedded CPU and GPU implementations, our FPGA implementation achieves up to 35.85x and 2.44x execution time speed up, respectively.

@inproceedings{
 title = {Energy-efficient FPGA-accelerated LiDAR-based SLAM for embedded robotics},
 type = {inproceedings},
 year = {2021},
 keywords = {Embedded LiDAR-based SLAM,FPGA acceleration,TSDF map,autonomous robotics},
 publisher = {Institute of Electrical and Electronics Engineers Inc.},
 id = {cd681ed2-46e1-3d37-b008-afed7dd28c90},
 created = {2025-06-03T16:20:26.667Z},
 file_attached = {true},
 profile_id = {78e67dcc-28e6-3300-a4ed-85434b13f01f},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2025-06-03T16:35:02.637Z},
 read = {true},
 starred = {true},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 notes = {13 cites  FPGA-accelerated LiDAR-based SLAM for embedded robotics, 3D mapping & tracking the robot's 6D pose  Xilinx <b>Vitis</b> and <b>HLS</b> tools, <b>Petalinux</b> on SoC FPGA, <b>ROS</b> viz & <b>Gazebo</b>, ZeroMQ Ethernet communication.   Xilinx XCZU15EG board},
 folder_uuids = {c9ff9cb6-d9de-44fb-a589-c1557950059d},
 private_publication = {false},
 abstract = {Being one of the fundamental problems in autonomous robotics, SLAM (Simultaneous Localization and Mapping) algorithms have gained a lot of attention. Although numerous approaches have been presented for determining 6D poses in 3D environments, one of the main challenges that remains is the required combination of real-time processing and high energy efficiency. In this paper, a combination of CPU and FPGA processing is used to tackle this problem, utilizing a reconfigurable SoC. We present a complete solution for embedded LiDAR-based SLAM that uses a global Truncated Signed Distance Function (TSDF) as map representation. A hardware-in-the-loop environment with ROS integration enables efficient evaluation of new variants of algorithms and implementations. Based on benchmark data sets and real-world environments, we show that our approach compares well to established SLAM algorithms. Compared to a software implementation on a state-of-the-art PC, the proposed implementation achieves a 7-fold speed-up and requires 18 times less energy when using a Xilinx UltraScale+ XCZU15EG.},
 bibtype = {inproceedings},
 author = {Flottmann, Marcel and Eisoldt, Marc and Gaal, Julian and Rothmann, Marc and Tassemeier, Marco and Wiemann, Thomas and Porrmann, Mario},
 doi = {10.1109/ICFPT52863.2021.9609934},
 booktitle = {2021 International Conference on Field-Programmable Technology, ICFPT 2021}
}

Being one of the fundamental problems in autonomous robotics, SLAM (Simultaneous Localization and Mapping) algorithms have gained a lot of attention. Although numerous approaches have been presented for determining 6D poses in 3D environments, one of the main challenges that remains is the required combination of real-time processing and high energy efficiency. In this paper, a combination of CPU and FPGA processing is used to tackle this problem, utilizing a reconfigurable SoC. We present a complete solution for embedded LiDAR-based SLAM that uses a global Truncated Signed Distance Function (TSDF) as map representation. A hardware-in-the-loop environment with ROS integration enables efficient evaluation of new variants of algorithms and implementations. Based on benchmark data sets and real-world environments, we show that our approach compares well to established SLAM algorithms. Compared to a software implementation on a state-of-the-art PC, the proposed implementation achieves a 7-fold speed-up and requires 18 times less energy when using a Xilinx UltraScale+ XCZU15EG.

@article{
 title = {From planes to corners: Multi-purpose primitive detection in unorganized 3D point clouds},
 type = {article},
 year = {2020},
 keywords = {Object detection,computational geometry,range sensing,segmentation and categori-zation},
 pages = {1764-1771},
 volume = {5},
 id = {77cff1a2-473c-3179-8b7a-8ece82645688},
 created = {2020-09-14T08:14:53.841Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-26T12:19:40.020Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Sommer2020},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,16688d52-1757-4ef4-badb-f53b700252a9,591145a4-49d3-4baf-a2cc-a1f3832f0e3e,beecb55d-84d0-48a2-a344-e50cfe559467,4f36a0a5-b08a-4f70-b020-4daf83cb0507},
 private_publication = {false},
 abstract = {We propose a new method for segmentation-free joint estimation of orthogonal planes, their intersection lines, relationship graph and corners lying at the intersection of three orthogonal planes. Such unified scene exploration under orthogonality allows for multitudes of applications such as semantic plane detection or local and global scan alignment, which in turn can aid robot localization or grasping tasks. Our two-stage pipeline involves a rough yet joint estimation of orthogonal planes followed by a subsequent joint refinement of plane parameters respecting their orthogonality relations. We form a graph of these primitives, paving the way to the extraction of further reliable features: lines and corners. Our experiments demonstrate the validity of our approach in numerous scenarios from wall detection to 6D tracking, both on synthetic and real data.},
 bibtype = {article},
 author = {Sommer, Christiane and Sun, Yumin and Guibas, Leonidas and Cremers, Daniel and Birdal, Tolga},
 doi = {10.1109/LRA.2020.2969936},
 journal = {IEEE Robotics and Automation Letters},
 number = {2}
}

We propose a new method for segmentation-free joint estimation of orthogonal planes, their intersection lines, relationship graph and corners lying at the intersection of three orthogonal planes. Such unified scene exploration under orthogonality allows for multitudes of applications such as semantic plane detection or local and global scan alignment, which in turn can aid robot localization or grasping tasks. Our two-stage pipeline involves a rough yet joint estimation of orthogonal planes followed by a subsequent joint refinement of plane parameters respecting their orthogonality relations. We form a graph of these primitives, paving the way to the extraction of further reliable features: lines and corners. Our experiments demonstrate the validity of our approach in numerous scenarios from wall detection to 6D tracking, both on synthetic and real data.

@article{
 title = {Geometric Primitives in LiDAR Point Clouds: A Review},
 type = {article},
 year = {2020},
 keywords = {Edges,geometric primitives,light detection and ranging (lidar),lines,planes,point clouds,regularization,skeletons,volumetric shapes},
 pages = {685-707},
 volume = {13},
 publisher = {IEEE},
 id = {29bc7c93-0a3a-3cb5-8f58-d7368cfa1cc7},
 created = {2020-09-14T08:14:53.877Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-01-25T14:53:38.134Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,16688d52-1757-4ef4-badb-f53b700252a9,591145a4-49d3-4baf-a2cc-a1f3832f0e3e,07e07de9-bcac-4934-a82b-d0aff540e56d},
 private_publication = {false},
 abstract = {To the best of our knowledge, the most recent light detection and ranging (lidar)-based surveys have been focused only on specific applications such as reconstruction and segmentation, as well as data processing techniques based on a specific platform, e.g., mobile laser. However, in this article, lidar point clouds are understood from a new and universal perspective, i.e., geometric primitives embedded in versatile objects in the physical world. In lidar point clouds, the basic unit is the point coordinate. Geometric primitives that consist of a group of discrete points may be viewed as one kind of abstraction and representation of lidar data at the entity level. We categorize geometric primitives into two classes: Shape primitives, e.g., lines, surfaces, and volumetric shapes, and structure primitives, represented by skeletons and edges. In recent years, many efforts from different communities, such as photogrammetry, computer vision, and computer graphics, have been made to finalize geometric primitive detection, regularization, and in-depth applications. Interpretations of geometric primitives from multiple disciplines try to convey the significance of geometric primitives, the latest processing techniques regarding geometric primitives, and their potential possibilities in the context of lidar point clouds. To this end, primitive-based applications are reviewed with an emphasis on object extraction and reconstruction to clearly show the significances of this article. Next, we survey and compare methods for geometric primitive extraction and then review primitive regularization methods that add real-world constrains to initial primitives. Finally, we summarize the challenges, expected applications, and describe possible future for primitive extraction methods that can achieve globally optimal results efficiently, even with disorganized, uneven, noisy, incomplete, and large-scale lidar point clouds.},
 bibtype = {article},
 author = {Xia, Shaobo and Chen, Dong and Wang, Ruisheng and Li, Jonathan and Zhang, Xinchang},
 doi = {10.1109/JSTARS.2020.2969119},
 journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}
}

To the best of our knowledge, the most recent light detection and ranging (lidar)-based surveys have been focused only on specific applications such as reconstruction and segmentation, as well as data processing techniques based on a specific platform, e.g., mobile laser. However, in this article, lidar point clouds are understood from a new and universal perspective, i.e., geometric primitives embedded in versatile objects in the physical world. In lidar point clouds, the basic unit is the point coordinate. Geometric primitives that consist of a group of discrete points may be viewed as one kind of abstraction and representation of lidar data at the entity level. We categorize geometric primitives into two classes: Shape primitives, e.g., lines, surfaces, and volumetric shapes, and structure primitives, represented by skeletons and edges. In recent years, many efforts from different communities, such as photogrammetry, computer vision, and computer graphics, have been made to finalize geometric primitive detection, regularization, and in-depth applications. Interpretations of geometric primitives from multiple disciplines try to convey the significance of geometric primitives, the latest processing techniques regarding geometric primitives, and their potential possibilities in the context of lidar point clouds. To this end, primitive-based applications are reviewed with an emphasis on object extraction and reconstruction to clearly show the significances of this article. Next, we survey and compare methods for geometric primitive extraction and then review primitive regularization methods that add real-world constrains to initial primitives. Finally, we summarize the challenges, expected applications, and describe possible future for primitive extraction methods that can achieve globally optimal results efficiently, even with disorganized, uneven, noisy, incomplete, and large-scale lidar point clouds.

@article{
 title = {A review on deep learning approaches for 3d data representations in retrieval and classifications},
 type = {article},
 year = {2020},
 keywords = {3D data representation,3D deep learning,3D models dataset,Classification,Computer vision,Retrieval},
 pages = {57566-57593},
 volume = {8},
 id = {666cc992-7b98-3881-842a-bab78b9176c6},
 created = {2020-09-14T08:14:53.886Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-03-31T07:21:16.792Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Gezawa2020},
 folder_uuids = {a89f4866-a7e8-4ea9-aa98-e3f470892f7c,2a0475f2-facb-4360-917f-00c5f8541f47},
 private_publication = {false},
 abstract = {Deep learning approach has been used extensively in image analysis tasks. However, implementing the methods in 3D data is a bit complex because most of the previously designed deep learning architectures used 1D or 2D as input. In this work, the performance of deep learning methods on different 3D data representations has been reviewed. Based on the categorization of the different 3D data representations proposed in this paper, the importance of choosing a suitable 3D data representation which depends on simplicity, usability, and efficiency has been highlighted. Furthermore, the origin and contents of the major 3D datasets were discussed in detail. Due to growing interest in 3D object retrieval and classification tasks, the performance of different 3D object retrieval and classification on ModelNet40 dataset were compared. According to the findings in this work, multi views methods surpass voxel-based methods and with increased layers and enough data augmentation the performance can still be increased. Therefore, it can be concluded that deep learning together with a suitable 3D data representation gives an effective approach for improving the performance of 3D shape analysis. Finally, some possible directions for future researches were suggested.},
 bibtype = {article},
 author = {Gezawa, Abubakar Sulaiman and Zhang, Yan and Wang, Qicong and Yunqi, Lei},
 doi = {10.1109/ACCESS.2020.2982196},
 journal = {IEEE Access}
}

Deep learning approach has been used extensively in image analysis tasks. However, implementing the methods in 3D data is a bit complex because most of the previously designed deep learning architectures used 1D or 2D as input. In this work, the performance of deep learning methods on different 3D data representations has been reviewed. Based on the categorization of the different 3D data representations proposed in this paper, the importance of choosing a suitable 3D data representation which depends on simplicity, usability, and efficiency has been highlighted. Furthermore, the origin and contents of the major 3D datasets were discussed in detail. Due to growing interest in 3D object retrieval and classification tasks, the performance of different 3D object retrieval and classification on ModelNet40 dataset were compared. According to the findings in this work, multi views methods surpass voxel-based methods and with increased layers and enough data augmentation the performance can still be increased. Therefore, it can be concluded that deep learning together with a suitable 3D data representation gives an effective approach for improving the performance of 3D shape analysis. Finally, some possible directions for future researches were suggested.

@article{
 title = {CNN-Based Lidar Point Cloud De-Noising in Adverse Weather},
 type = {article},
 year = {2020},
 keywords = {Semantic scene understanding,computer vision for transportation,visual learning},
 pages = {2514-2521},
 volume = {5},
 id = {4eb6e6f2-883b-3eaa-8d2c-3bfb29487798},
 created = {2020-10-01T13:48:35.420Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-10-01T13:48:55.548Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {6533efe4-7189-42a2-b4b6-a9f175595b19},
 private_publication = {false},
 abstract = {Lidar sensors are frequently used in environment perception for autonomous vehicles and mobile robotics to complement camera, radar, and ultrasonic sensors. Adverse weather conditions are significantly impacting the performance of lidar-based scene understanding by causing undesired measurement points that in turn effect missing detections and false positives. In heavy rain or dense fog, water drops could be misinterpreted as objects in front of the vehicle which brings a mobile robot to a full stop. In this letter, we present the first CNN-based approach to understand and filter out such adverse weather effects in point cloud data. Using a large data set obtained in controlled weather environments, we demonstrate a significant performance improvement of our method over state-of-the-art involving geometric filtering. Data is available at https://github.com/rheinzler/PointCloudDeNoising.},
 bibtype = {article},
 author = {Heinzler, Robin and Piewak, Florian and Schindler, Philipp and Stork, Wilhelm},
 doi = {10.1109/LRA.2020.2972865},
 journal = {IEEE Robotics and Automation Letters},
 number = {2}
}

Lidar sensors are frequently used in environment perception for autonomous vehicles and mobile robotics to complement camera, radar, and ultrasonic sensors. Adverse weather conditions are significantly impacting the performance of lidar-based scene understanding by causing undesired measurement points that in turn effect missing detections and false positives. In heavy rain or dense fog, water drops could be misinterpreted as objects in front of the vehicle which brings a mobile robot to a full stop. In this letter, we present the first CNN-based approach to understand and filter out such adverse weather effects in point cloud data. Using a large data set obtained in controlled weather environments, we demonstrate a significant performance improvement of our method over state-of-the-art involving geometric filtering. Data is available at https://github.com/rheinzler/PointCloudDeNoising.

@article{
 title = {Supervised learning of the next-best-view for 3d object reconstruction},
 type = {article},
 year = {2020},
 keywords = {3D reconstruction,3D-CNN,Next-best-view},
 pages = {224-231},
 volume = {133},
 websites = {https://doi.org/10.1016/j.patrec.2020.02.024},
 publisher = {Elsevier B.V.},
 id = {9ce275d2-18aa-3157-994c-d5dc9e2a7ab2},
 created = {2020-10-05T10:26:00.758Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-26T12:19:39.858Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Mendoza2020},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,bc26f4dd-ccfc-4a52-b602-2ceb657d0906,07e07de9-bcac-4934-a82b-d0aff540e56d,4f36a0a5-b08a-4f70-b020-4daf83cb0507},
 private_publication = {false},
 abstract = {Motivated by the advances in 3D sensing technology and the spreading of low-cost robotic platforms, 3D object reconstruction has become a common task in many areas. Nevertheless, the selection of the optimal sensor pose that maximizes the reconstructed surface is a problem that remains open. It is known in the literature as the next-best-view planning problem. In this paper, we propose a novel next-best-view planning scheme based on supervised deep learning. The scheme contains an algorithm for automatic generation of datasets and an original three-dimensional convolutional neural network (3D-CNN) used to learn the next-best-view. Unlike previous work where the problem is addressed as a search, the trained 3D-CNN directly predicts the sensor pose. We present an experimental comparison of the proposed architecture against two alternative networks; we also compare it with state-of-the-art next-best-view methods in the reconstruction of several unknown objects. Our method is faster and reaches high coverage.},
 bibtype = {article},
 author = {Mendoza, Miguel and Vasquez-Gomez, J. Irving and Taud, Hind and Sucar, L. Enrique and Reta, Carolina},
 doi = {10.1016/j.patrec.2020.02.024},
 journal = {Pattern Recognition Letters}
}

Motivated by the advances in 3D sensing technology and the spreading of low-cost robotic platforms, 3D object reconstruction has become a common task in many areas. Nevertheless, the selection of the optimal sensor pose that maximizes the reconstructed surface is a problem that remains open. It is known in the literature as the next-best-view planning problem. In this paper, we propose a novel next-best-view planning scheme based on supervised deep learning. The scheme contains an algorithm for automatic generation of datasets and an original three-dimensional convolutional neural network (3D-CNN) used to learn the next-best-view. Unlike previous work where the problem is addressed as a search, the trained 3D-CNN directly predicts the sensor pose. We present an experimental comparison of the proposed architecture against two alternative networks; we also compare it with state-of-the-art next-best-view methods in the reconstruction of several unknown objects. Our method is faster and reaches high coverage.

@article{
 title = {PointContrast: Unsupervised Pre-training for 3D Point Cloud Understanding},
 type = {article},
 year = {2020},
 keywords = {3d scene understanding,point cloud recognition,repre-,sentation learning,unsupervised learning},
 websites = {http://arxiv.org/abs/2007.10985},
 id = {d77a7559-98f3-3789-b3ac-6e00a4f76049},
 created = {2020-10-15T09:39:12.560Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-01-28T08:25:31.061Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {a89f4866-a7e8-4ea9-aa98-e3f470892f7c},
 private_publication = {false},
 abstract = {Arguably one of the top success stories of deep learning is transfer learning. The finding that pre-training a network on a rich source set (eg., ImageNet) can help boost performance once fine-tuned on a usually much smaller target set, has been instrumental to many applications in language and vision. Yet, very little is known about its usefulness in 3D point cloud understanding. We see this as an opportunity considering the effort required for annotating data in 3D. In this work, we aim at facilitating research on 3D representation learning. Different from previous works, we focus on high-level scene understanding tasks. To this end, we select a suite of diverse datasets and tasks to measure the effect of unsupervised pre-training on a large source set of 3D scenes. Our findings are extremely encouraging: using a unified triplet of architecture, source dataset, and contrastive loss for pre-training, we achieve improvement over recent best results in segmentation and detection across 6 different benchmarks for indoor and outdoor, real and synthetic datasets -- demonstrating that the learned representation can generalize across domains. Furthermore, the improvement was similar to supervised pre-training, suggesting that future efforts should favor scaling data collection over more detailed annotation. We hope these findings will encourage more research on unsupervised pretext task design for 3D deep learning.},
 bibtype = {article},
 author = {Xie, Saining and Gu, Jiatao and Guo, Demi and Qi, Charles R. and Guibas, Leonidas J. and Litany, Or}
}

Arguably one of the top success stories of deep learning is transfer learning. The finding that pre-training a network on a rich source set (eg., ImageNet) can help boost performance once fine-tuned on a usually much smaller target set, has been instrumental to many applications in language and vision. Yet, very little is known about its usefulness in 3D point cloud understanding. We see this as an opportunity considering the effort required for annotating data in 3D. In this work, we aim at facilitating research on 3D representation learning. Different from previous works, we focus on high-level scene understanding tasks. To this end, we select a suite of diverse datasets and tasks to measure the effect of unsupervised pre-training on a large source set of 3D scenes. Our findings are extremely encouraging: using a unified triplet of architecture, source dataset, and contrastive loss for pre-training, we achieve improvement over recent best results in segmentation and detection across 6 different benchmarks for indoor and outdoor, real and synthetic datasets -- demonstrating that the learned representation can generalize across domains. Furthermore, the improvement was similar to supervised pre-training, suggesting that future efforts should favor scaling data collection over more detailed annotation. We hope these findings will encourage more research on unsupervised pretext task design for 3D deep learning.

@article{
 title = {A survey on indoor RGB-D semantic segmentation : from hand-crafted features to deep convolutional neural networks},
 type = {article},
 year = {2020},
 keywords = {RGB-Depth images,Semantic segmentation,Deep learni,deep learning,hand-crafted,rgb-depth images,semantic segmentation},
 pages = {4499-4524},
 publisher = {Multimedia Tools and Applications},
 id = {d1ad6612-e4fa-3f0d-b57a-ae0ff4641411},
 created = {2020-10-20T09:48:06.186Z},
 file_attached = {true},
 profile_id = {235249c2-3ed4-314a-b309-b1ea0330f5d9},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-07-20T14:11:56.184Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 citation_key = {Fooladgar2020},
 folder_uuids = {dc009c1c-5c21-43bd-9c8a-d37db3983b2e,a89f4866-a7e8-4ea9-aa98-e3f470892f7c,5cd4d7ce-c2fb-4e91-ab80-35deeb123df5},
 private_publication = {false},
 bibtype = {article},
 author = {Fooladgar, Fahimeh and Kasaei, Shohreh}
}

@article{
 title = {Uncertainty-Aware CNNs for Depth Completion: Uncertainty from Beginning to End},
 type = {article},
 year = {2020},
 pages = {12011-12020},
 id = {acce76bf-829c-3c11-8681-ca1bdacb39ef},
 created = {2020-10-22T05:44:10.553Z},
 file_attached = {true},
 profile_id = {f3d36c73-062b-3738-9a74-d09e4e83eb1e},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-10-22T05:44:15.535Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {2a0475f2-facb-4360-917f-00c5f8541f47},
 private_publication = {false},
 abstract = {The focus in deep learning research has been mostly to push the limits of prediction accuracy. However, this was often achieved at the cost of increased complexity, raising concerns about the interpretability and the reliability of deep networks. Recently, an increasing attention has been given to untangling the complexity of deep networks and quantifying their uncertainty for different computer vision tasks. Differently, the task of depth completion has not received enough attention despite the inherent noisy nature of depth sensors. In this work, we thus focus on modeling the uncertainty of depth data in depth completion starting from the sparse noisy input all the way to the final prediction. We propose a novel approach to identify disturbed measurements in the input by learning an input confidence estimator in a self-supervised manner based on the normalized convolutional neural networks (NCNNs). Further, we propose a probabilistic version of NCNNs that produces a statistically meaningful uncertainty measure for the final prediction. When we evaluate our approach on the KITTI dataset for depth completion, we outperform all the existing Bayesian Deep Learning approaches in terms of prediction accuracy, quality of the uncertainty measure, and the computational efficiency. Moreover, our small network with 670k parameters performs on-par with conventional approaches with millions of parameters. These results give strong evidence that separating the network into parallel uncertainty and prediction streams leads to state-of-the-art performance with accurate uncertainty estimates.},
 bibtype = {article},
 author = {Eldesokey, Abdelrahman and Felsberg, Michael and Holmquist, Karl and Persson, Michael},
 doi = {10.1109/cvpr42600.2020.01203}
}

The focus in deep learning research has been mostly to push the limits of prediction accuracy. However, this was often achieved at the cost of increased complexity, raising concerns about the interpretability and the reliability of deep networks. Recently, an increasing attention has been given to untangling the complexity of deep networks and quantifying their uncertainty for different computer vision tasks. Differently, the task of depth completion has not received enough attention despite the inherent noisy nature of depth sensors. In this work, we thus focus on modeling the uncertainty of depth data in depth completion starting from the sparse noisy input all the way to the final prediction. We propose a novel approach to identify disturbed measurements in the input by learning an input confidence estimator in a self-supervised manner based on the normalized convolutional neural networks (NCNNs). Further, we propose a probabilistic version of NCNNs that produces a statistically meaningful uncertainty measure for the final prediction. When we evaluate our approach on the KITTI dataset for depth completion, we outperform all the existing Bayesian Deep Learning approaches in terms of prediction accuracy, quality of the uncertainty measure, and the computational efficiency. Moreover, our small network with 670k parameters performs on-par with conventional approaches with millions of parameters. These results give strong evidence that separating the network into parallel uncertainty and prediction streams leads to state-of-the-art performance with accurate uncertainty estimates.

@article{
 title = {Normal Estimation for 3D Point Clouds via Local Plane Constraint and Multi-scale Selection},
 type = {article},
 year = {2020},
 keywords = {LPFC,Multi-scale selection,Normal estimation,Plane-aware features,Point cloud processing,Robust to noise},
 volume = {129},
 id = {f2300a44-7d4e-33ca-b450-9035b4872354},
 created = {2020-11-03T13:16:20.066Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-02-24T14:09:30.067Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Zhou2020},
 folder_uuids = {71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {In this paper, we propose a normal estimation method for unstructured 3D point clouds. In this method, a feature constraint mechanism called Local Plane Features Constraint (LPFC) is used and then a multi-scale selection strategy is introduced. The LPFC can be used in a single-scale point network architecture for a more stable normal estimation of the unstructured 3D point clouds. In particular, it can partly overcome the influence of noise on a large sampling scale compared to the other methods which only use regression loss for normal estimation. For more details, a subnetwork is built after point-wise features extracted layers of the network and it gives more constraints to each point of the local patch via a binary classifier in the end. Then we use multi-task optimization to train the normal estimation and local plane classification tasks simultaneously. Via LPFC, the normal estimation network could obtain more distinguish point-wise plane-aware features that can describe the differences of each point on the local patch. Finally, thanks to the distinguish features constraint, we can obtain a more robust and meaningful global feature that can be used to regress the normal of the local patch. Also, to integrate the advantages of multi-scale results, a scale selection strategy is adopted, which is a data-driven approach for selecting the optimal scale around each point and encourages subnetwork specialization. Specifically, we employed a subnetwork called Scale Estimation Network to extract scale weight information from multi-scale features. The multi-scale method can well reduce the cost time while persevere the estimation accuracy. More analysis is given about the relations between noise levels, local boundary, and scales in the experiment. These relationships can be a better guide to choosing particular scales for a particular model. Besides, the experimental result shows that our network can distinguish the points on the fitting plane accurately and this can be used to guide the normal estimation and our multi-scale method can improve the results well. Compared to some state-of-the-art surface normal estimators, our method is robust to noise and can achieve competitive results.},
 bibtype = {article},
 author = {Zhou, Jun and Huang, Hua and Liu, Bin and Liu, Xiuping},
 doi = {10.1016/j.cad.2020.102916},
 journal = {CAD Computer Aided Design}
}

In this paper, we propose a normal estimation method for unstructured 3D point clouds. In this method, a feature constraint mechanism called Local Plane Features Constraint (LPFC) is used and then a multi-scale selection strategy is introduced. The LPFC can be used in a single-scale point network architecture for a more stable normal estimation of the unstructured 3D point clouds. In particular, it can partly overcome the influence of noise on a large sampling scale compared to the other methods which only use regression loss for normal estimation. For more details, a subnetwork is built after point-wise features extracted layers of the network and it gives more constraints to each point of the local patch via a binary classifier in the end. Then we use multi-task optimization to train the normal estimation and local plane classification tasks simultaneously. Via LPFC, the normal estimation network could obtain more distinguish point-wise plane-aware features that can describe the differences of each point on the local patch. Finally, thanks to the distinguish features constraint, we can obtain a more robust and meaningful global feature that can be used to regress the normal of the local patch. Also, to integrate the advantages of multi-scale results, a scale selection strategy is adopted, which is a data-driven approach for selecting the optimal scale around each point and encourages subnetwork specialization. Specifically, we employed a subnetwork called Scale Estimation Network to extract scale weight information from multi-scale features. The multi-scale method can well reduce the cost time while persevere the estimation accuracy. More analysis is given about the relations between noise levels, local boundary, and scales in the experiment. These relationships can be a better guide to choosing particular scales for a particular model. Besides, the experimental result shows that our network can distinguish the points on the fitting plane accurately and this can be used to guide the normal estimation and our multi-scale method can improve the results well. Compared to some state-of-the-art surface normal estimators, our method is robust to noise and can achieve competitive results.

@article{
 title = {PointCleanNet: Learning to Denoise and Remove Outliers from Dense Point Clouds},
 type = {article},
 year = {2020},
 keywords = {Neural networks,Shape analysis,[Computing Methodologies]: Point-based models,methods and applications,modeling,point-based graphics,point-based methods,signal processing},
 pages = {185-203},
 volume = {39},
 id = {8d867188-3ef7-3b02-b746-ef3fa43dbd45},
 created = {2020-11-03T13:16:20.067Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-03-19T07:55:50.086Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Rakotosaona2020},
 folder_uuids = {ff4ee14e-f1de-4f3e-889c-95d41b8d7277},
 private_publication = {false},
 abstract = {Point clouds obtained with 3D scanners or by image-based reconstruction techniques are often corrupted with significant amount of noise and outliers. Traditional methods for point cloud denoising largely rely on local surface fitting (e.g. jets or MLS surfaces), local or non-local averaging or on statistical assumptions about the underlying noise model. In contrast, we develop a simple data-driven method for removing outliers and reducing noise in unordered point clouds. We base our approach on a deep learning architecture adapted from PCPNet, which was recently proposed for estimating local 3D shape properties in point clouds. Our method first classifies and discards outlier samples, and then estimates correction vectors that project noisy points onto the original clean surfaces. The approach is efficient and robust to varying amounts of noise and outliers, while being able to handle large densely sampled point clouds. In our extensive evaluation, both on synthetic and real data, we show an increased robustness to strong noise levels compared to various state-of-the-art methods, enabling accurate surface reconstruction from extremely noisy real data obtained by range scans. Finally, the simplicity and universality of our approach makes it very easy to integrate in any existing geometry processing pipeline. Both the code and pre-trained networks can be found on the project page (https://github.com/mrakotosaon/pointcleannet).},
 bibtype = {article},
 author = {Rakotosaona, Marie Julie and La Barbera, Vittorio and Guerrero, Paul and Mitra, Niloy J. and Ovsjanikov, Maks},
 doi = {10.1111/cgf.13753},
 journal = {Computer Graphics Forum},
 number = {1}
}

Point clouds obtained with 3D scanners or by image-based reconstruction techniques are often corrupted with significant amount of noise and outliers. Traditional methods for point cloud denoising largely rely on local surface fitting (e.g. jets or MLS surfaces), local or non-local averaging or on statistical assumptions about the underlying noise model. In contrast, we develop a simple data-driven method for removing outliers and reducing noise in unordered point clouds. We base our approach on a deep learning architecture adapted from PCPNet, which was recently proposed for estimating local 3D shape properties in point clouds. Our method first classifies and discards outlier samples, and then estimates correction vectors that project noisy points onto the original clean surfaces. The approach is efficient and robust to varying amounts of noise and outliers, while being able to handle large densely sampled point clouds. In our extensive evaluation, both on synthetic and real data, we show an increased robustness to strong noise levels compared to various state-of-the-art methods, enabling accurate surface reconstruction from extremely noisy real data obtained by range scans. Finally, the simplicity and universality of our approach makes it very easy to integrate in any existing geometry processing pipeline. Both the code and pre-trained networks can be found on the project page (https://github.com/mrakotosaon/pointcleannet).

@article{
 title = {Point Cloud Normal Estimation by Fast Guided Least Squares Representation},
 type = {article},
 year = {2020},
 keywords = {Normal estimation,fast algorithm,feature preserving,least squares representation},
 pages = {101580-101590},
 volume = {8},
 id = {231a8f25-4b70-3993-8913-2848f1456fcf},
 created = {2020-11-03T13:16:20.072Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-11-03T13:17:39.505Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {Normal estimation is an essential task for scanned point clouds in various CAD/CAM applications. The method (GLSRNE) based on guided least squares representation (GLSR) balances speed with quality well among state-of-the-art methods. First, it segments each neighborhood into multiple sub-neighborhoods. For some neighborhoods, the segmentation is obtained by GLSR which is an efficient subspace segmentation model and widely applied in other applications. The segmentation of the rest neighborhoods is inferred via the subspace structure propagation (SSP) algorithm. Then, each sub-neighborhood is fitted by a plane. The plane achieving the minimum distance with the current point is selected for the final normal estimation. We make improvements for effectiveness and efficiency in the following three aspects. First, to improve the speed of GLSR, we propose a novel iterative algorithm to reduce the computation complexity from $O(n^3)$ to $O(n^2)$ with its convergence guaranteed theoretically, where $n$ represents the number of the data points. Moreover, this proposed algorithm will also be useful for other applications. Second, we add a normal constraint for SSP to improve accuracy. Third, when selecting one plane to estimate the final normal, we consider the match between the plane and all neighbors, whereas GLSRNE only considers the match between the plane and the current point. The experiments exhibit that our method is faster than GLSRNE and more effective than GLSRNE and other state-of-the-art methods.},
 bibtype = {article},
 author = {Zhang, Jie and Duan, Jiahui and Tang, Kewei and Cao, Junjie and Liu, Xiuping},
 doi = {10.1109/ACCESS.2020.2998468},
 journal = {IEEE Access}
}

Normal estimation is an essential task for scanned point clouds in various CAD/CAM applications. The method (GLSRNE) based on guided least squares representation (GLSR) balances speed with quality well among state-of-the-art methods. First, it segments each neighborhood into multiple sub-neighborhoods. For some neighborhoods, the segmentation is obtained by GLSR which is an efficient subspace segmentation model and widely applied in other applications. The segmentation of the rest neighborhoods is inferred via the subspace structure propagation (SSP) algorithm. Then, each sub-neighborhood is fitted by a plane. The plane achieving the minimum distance with the current point is selected for the final normal estimation. We make improvements for effectiveness and efficiency in the following three aspects. First, to improve the speed of GLSR, we propose a novel iterative algorithm to reduce the computation complexity from $O(n^3)$ to $O(n^2)$ with its convergence guaranteed theoretically, where $n$ represents the number of the data points. Moreover, this proposed algorithm will also be useful for other applications. Second, we add a normal constraint for SSP to improve accuracy. Third, when selecting one plane to estimate the final normal, we consider the match between the plane and all neighbors, whereas GLSRNE only considers the match between the plane and the current point. The experiments exhibit that our method is faster than GLSRNE and more effective than GLSRNE and other state-of-the-art methods.

@article{
 title = {Confidence Estimation for ToF and Stereo Sensors and Its Application to Depth Data Fusion},
 type = {article},
 year = {2020},
 keywords = {Time-of-flight,confidence information,data fusion,deep learning,stereo vision},
 pages = {1411-1421},
 volume = {20},
 id = {7c7ca9cd-dcc3-3c52-aff0-57a6b4c53bf6},
 created = {2020-11-05T09:10:48.240Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-11-11T06:38:13.097Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {2a0475f2-facb-4360-917f-00c5f8541f47},
 private_publication = {false},
 abstract = {Time-of-Flight (ToF) sensors and stereo vision systems are two widely used technologies for depth estimation. Due to their rather complementary strengths and limitations, the two sensors are often combined to infer more accurate depth maps. A key research issue in this field is how to estimate the reliability of the sensed depth data. While this problem has been widely studied for stereo systems, it has been seldom considered for ToF sensors. Therefore, starting from the work done for stereo data, in this paper, we firstly introduce novel confidence estimation techniques for ToF data. Moreover, we also show how by using learning-based confidence metrics jointly trained on the two sensors yields better performance. Finally, deploying different fusion frameworks, we show how confidence estimation can be exploited in order to guide the fusion of depth data from the two sensors. Experimental results show how accurate confidence cues allow outperforming state-of-the-art data fusion schemes even with the simplest fusion strategies known in the literature.},
 bibtype = {article},
 author = {Poggi, Matteo and Agresti, Gianluca and Tosi, Fabio and Zanuttigh, Pietro and Mattoccia, Stefano},
 doi = {10.1109/JSEN.2019.2946591},
 journal = {IEEE Sensors Journal},
 number = {3}
}

Time-of-Flight (ToF) sensors and stereo vision systems are two widely used technologies for depth estimation. Due to their rather complementary strengths and limitations, the two sensors are often combined to infer more accurate depth maps. A key research issue in this field is how to estimate the reliability of the sensed depth data. While this problem has been widely studied for stereo systems, it has been seldom considered for ToF sensors. Therefore, starting from the work done for stereo data, in this paper, we firstly introduce novel confidence estimation techniques for ToF data. Moreover, we also show how by using learning-based confidence metrics jointly trained on the two sensors yields better performance. Finally, deploying different fusion frameworks, we show how confidence estimation can be exploited in order to guide the fusion of depth data from the two sensors. Experimental results show how accurate confidence cues allow outperforming state-of-the-art data fusion schemes even with the simplest fusion strategies known in the literature.

@article{
 title = {Learning to segment 3D point clouds in 2D image space},
 type = {article},
 year = {2020},
 pages = {12252-12261},
 id = {11eaa7ce-3478-3eb6-a788-ce0ed8f23361},
 created = {2020-11-13T11:34:36.717Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-11-13T11:36:34.913Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {In contrast to the literature where local patterns in 3D point clouds are captured by customized convolutional operators, in this paper we study the problem of how to effectively and efficiently project such point clouds into a 2D image space so that traditional 2D convolutional neural networks (CNNs) such as U-Net can be applied for segmentation. To this end, we are motivated by graph drawing and reformulate it as an integer programming problem to learn the topology-preserving graph-to-grid mapping for each individual point cloud. To accelerate the computation in practice, we further propose a novel hierarchical approximate algorithm. With the help of the Delaunay triangulation for graph construction from point clouds and a multi-scale U-Net for segmentation, we manage to demonstrate the state-of-the-art performance on ShapeNet and PartNet, respectively, with significant improvement over the literature. Code is available at https://github.com/Zhang-VISLab.},
 bibtype = {article},
 author = {Lyu, Yecheng and Huang, Xinming and Zhang, Ziming},
 doi = {10.1109/CVPR42600.2020.01227},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

In contrast to the literature where local patterns in 3D point clouds are captured by customized convolutional operators, in this paper we study the problem of how to effectively and efficiently project such point clouds into a 2D image space so that traditional 2D convolutional neural networks (CNNs) such as U-Net can be applied for segmentation. To this end, we are motivated by graph drawing and reformulate it as an integer programming problem to learn the topology-preserving graph-to-grid mapping for each individual point cloud. To accelerate the computation in practice, we further propose a novel hierarchical approximate algorithm. With the help of the Delaunay triangulation for graph construction from point clouds and a multi-scale U-Net for segmentation, we manage to demonstrate the state-of-the-art performance on ShapeNet and PartNet, respectively, with significant improvement over the literature. Code is available at https://github.com/Zhang-VISLab.

@article{
 title = {Normal assisted stereo depth estimation},
 type = {article},
 year = {2020},
 pages = {2186-2196},
 id = {c525dc05-ec38-3875-913d-c1480d7676e5},
 created = {2020-11-13T11:34:36.721Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-11-13T11:36:04.070Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {Accurate stereo depth estimation plays a critical role in various 3D tasks in both indoor and outdoor environments. Recently, learning-based multi-view stereo methods have demonstrated competitive performance with limited number of views. However, in challenging scenarios, especially when building cross-view correspondences is hard, these methods still cannot produce satisfying results. In this paper, we study how to leverage a normal estimation model and the predicted normal maps to improve the depth quality. We couple the learning of a multi-view normal estimation module and a multi-view depth estimation module. In addition, we propose a novel consistency loss to train an independent consistency module that refines the depths from depth/normal pairs. We find that the joint learning can improve both the prediction of normal and depth, and the accuracy & smoothness can be further improved by enforcing the consistency. Experiments on MVS, SUN3D, RGBD and Scenes11 demonstrate the effectiveness of our method and state-of-the-art performance.},
 bibtype = {article},
 author = {Kusupati, Uday and Cheng, Shuo and Chen, Rui and Su, Hao},
 doi = {10.1109/CVPR42600.2020.00226},
 journal = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition}
}

Accurate stereo depth estimation plays a critical role in various 3D tasks in both indoor and outdoor environments. Recently, learning-based multi-view stereo methods have demonstrated competitive performance with limited number of views. However, in challenging scenarios, especially when building cross-view correspondences is hard, these methods still cannot produce satisfying results. In this paper, we study how to leverage a normal estimation model and the predicted normal maps to improve the depth quality. We couple the learning of a multi-view normal estimation module and a multi-view depth estimation module. In addition, we propose a novel consistency loss to train an independent consistency module that refines the depths from depth/normal pairs. We find that the joint learning can improve both the prediction of normal and depth, and the accuracy & smoothness can be further improved by enforcing the consistency. Experiments on MVS, SUN3D, RGBD and Scenes11 demonstrate the effectiveness of our method and state-of-the-art performance.

@article{
 title = {Neighbourhood-Insensitive Point Cloud Normal Estimation Network},
 type = {article},
 year = {2020},
 websites = {http://arxiv.org/abs/2008.09965},
 id = {242d02a0-b8f0-37df-a222-caad9bb43942},
 created = {2020-11-13T11:34:37.182Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-11-13T11:34:58.643Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {We introduce a novel self-attention-based normal estimation network that is able to focus softly on relevant points and adjust the softness by learning a temperature parameter, making it able to work naturally and effectively within a large neighbourhood range. As a result, our model outperforms all existing normal estimation algorithms by a large margin, achieving 94.1% accuracy in comparison with the previous state of the art of 91.2%, with a 25x smaller model and 12x faster inference time. We also use point-to-plane Iterative Closest Point (ICP) as an application case to show that our normal estimations lead to faster convergence than normal estimations from other methods, without manually fine-tuning neighbourhood range parameters. Code available at https://code.active.vision.},
 bibtype = {article},
 author = {Wang, Zirui and Prisacariu, Victor Adrian}
}

We introduce a novel self-attention-based normal estimation network that is able to focus softly on relevant points and adjust the softness by learning a temperature parameter, making it able to work naturally and effectively within a large neighbourhood range. As a result, our model outperforms all existing normal estimation algorithms by a large margin, achieving 94.1% accuracy in comparison with the previous state of the art of 91.2%, with a 25x smaller model and 12x faster inference time. We also use point-to-plane Iterative Closest Point (ICP) as an application case to show that our normal estimations lead to faster convergence than normal estimations from other methods, without manually fine-tuning neighbourhood range parameters. Code available at https://code.active.vision.

@article{
 title = {Deep learning based multi-modal fusion architectures for maritime vessel detection},
 type = {article},
 year = {2020},
 keywords = {Autonomous vehicles,Convolutional neural networks,Deep learning,Marine environment,Multi-sensor fusion,Object detection},
 volume = {12},
 id = {124db671-ca53-3b85-9ed8-6450ffb6eac2},
 created = {2020-11-16T11:56:20.677Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-02-24T11:29:17.832Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {f3937f49-b3bb-4fad-a335-4fb2125beca8,71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 abstract = {Object detection is a fundamental computer vision task for many real-world applications. In the maritime environment, this task is challenging due to varying light, view distances, weather conditions, and sea waves. In addition, light reflection, camera motion and illumination changes may cause to false detections. To address this challenge, we present three fusion architectures to fuse two imaging modalities: visible and infrared. These architectures can provide complementary information from two modalities in different levels: pixel-level, feature-level, and decision-level. They employed deep learning for performing fusion and detection. We investigate the performance of the proposed architectures conducting a real marine image dataset, which is captured by color and infrared cameras on-board a vessel in the Finnish archipelago. The cameras are employed for developing autonomous ships, and collect data in a range of operation and climatic conditions. Experiments show that feature-level fusion architecture outperforms the state-of-the-art other fusion level architectures.},
 bibtype = {article},
 author = {Farahnakian, Fahimeh and Heikkonen, Jukka},
 doi = {10.3390/RS12162509},
 journal = {Remote Sensing},
 number = {16}
}

Object detection is a fundamental computer vision task for many real-world applications. In the maritime environment, this task is challenging due to varying light, view distances, weather conditions, and sea waves. In addition, light reflection, camera motion and illumination changes may cause to false detections. To address this challenge, we present three fusion architectures to fuse two imaging modalities: visible and infrared. These architectures can provide complementary information from two modalities in different levels: pixel-level, feature-level, and decision-level. They employed deep learning for performing fusion and detection. We investigate the performance of the proposed architectures conducting a real marine image dataset, which is captured by color and infrared cameras on-board a vessel in the Finnish archipelago. The cameras are employed for developing autonomous ships, and collect data in a range of operation and climatic conditions. Experiments show that feature-level fusion architecture outperforms the state-of-the-art other fusion level architectures.

@article{
 title = {CNN based Color and Thermal Image Fusion for Object Detection in CNN based Color and Thermal Image Fusion for Object Detection in Automated Driving},
 type = {article},
 year = {2020},
 id = {03d96ec2-97ae-3b73-808f-d42784f579f5},
 created = {2020-11-16T11:56:20.702Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-02-24T11:29:17.247Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {f3937f49-b3bb-4fad-a335-4fb2125beca8,71ca8421-f528-4caf-a342-3c1291372174},
 private_publication = {false},
 bibtype = {article},
 author = {Yadav, Ravi and Samir, Ahmed and Rashed, Hazem and Yogamani, Senthil and Dahyot, Rozenn},
 number = {July}
}

@article{
 title = {From Planes to Corners : Multi-Purpose Primitive Detection in Unorganized 3D Point Clouds},
 type = {article},
 year = {2020},
 pages = {1-8},
 id = {b058cafd-237b-3011-b69c-b63be4a8238a},
 created = {2020-12-11T09:32:46.150Z},
 file_attached = {true},
 profile_id = {bfbbf840-4c42-3914-a463-19024f50b30c},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2020-12-11T09:33:36.131Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {false},
 hidden = {false},
 folder_uuids = {591145a4-49d3-4baf-a2cc-a1f3832f0e3e},
 private_publication = {false},
 bibtype = {article},
 author = {Sommer, Christiane and Guibas, Leonidas and Cremers, Daniel},
 number = {c}
}

@article{
 title = {Geometric Primitives in LiDAR Point Clouds: A Review},
 type = {article},
 year = {2020},
 keywords = {Edges,geometric primitives,light detection and ranging (lidar),lines,planes,point clouds,regularization,skeletons,volumetric shapes},
 pages = {685-707},
 volume = {13},
 id = {33654dff-2635-33a7-bc70-929efd158d04},
 created = {2021-01-25T14:53:33.640Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-02-09T17:05:54.920Z},
 read = {true},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,beecb55d-84d0-48a2-a344-e50cfe559467},
 private_publication = {false},
 abstract = {To the best of our knowledge, the most recent light detection and ranging (lidar)-based surveys have been focused only on specific applications such as reconstruction and segmentation, as well as data processing techniques based on a specific platform, e.g., mobile laser. However, in this article, lidar point clouds are understood from a new and universal perspective, i.e., geometric primitives embedded in versatile objects in the physical world. In lidar point clouds, the basic unit is the point coordinate. Geometric primitives that consist of a group of discrete points may be viewed as one kind of abstraction and representation of lidar data at the entity level. We categorize geometric primitives into two classes: Shape primitives, e.g., lines, surfaces, and volumetric shapes, and structure primitives, represented by skeletons and edges. In recent years, many efforts from different communities, such as photogrammetry, computer vision, and computer graphics, have been made to finalize geometric primitive detection, regularization, and in-depth applications. Interpretations of geometric primitives from multiple disciplines try to convey the significance of geometric primitives, the latest processing techniques regarding geometric primitives, and their potential possibilities in the context of lidar point clouds. To this end, primitive-based applications are reviewed with an emphasis on object extraction and reconstruction to clearly show the significances of this article. Next, we survey and compare methods for geometric primitive extraction and then review primitive regularization methods that add real-world constrains to initial primitives. Finally, we summarize the challenges, expected applications, and describe possible future for primitive extraction methods that can achieve globally optimal results efficiently, even with disorganized, uneven, noisy, incomplete, and large-scale lidar point clouds.},
 bibtype = {article},
 author = {Xia, Shaobo and Chen, Dong and Wang, Ruisheng and Li, Jonathan and Zhang, Xinchang},
 doi = {10.1109/JSTARS.2020.2969119},
 journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}
}

To the best of our knowledge, the most recent light detection and ranging (lidar)-based surveys have been focused only on specific applications such as reconstruction and segmentation, as well as data processing techniques based on a specific platform, e.g., mobile laser. However, in this article, lidar point clouds are understood from a new and universal perspective, i.e., geometric primitives embedded in versatile objects in the physical world. In lidar point clouds, the basic unit is the point coordinate. Geometric primitives that consist of a group of discrete points may be viewed as one kind of abstraction and representation of lidar data at the entity level. We categorize geometric primitives into two classes: Shape primitives, e.g., lines, surfaces, and volumetric shapes, and structure primitives, represented by skeletons and edges. In recent years, many efforts from different communities, such as photogrammetry, computer vision, and computer graphics, have been made to finalize geometric primitive detection, regularization, and in-depth applications. Interpretations of geometric primitives from multiple disciplines try to convey the significance of geometric primitives, the latest processing techniques regarding geometric primitives, and their potential possibilities in the context of lidar point clouds. To this end, primitive-based applications are reviewed with an emphasis on object extraction and reconstruction to clearly show the significances of this article. Next, we survey and compare methods for geometric primitive extraction and then review primitive regularization methods that add real-world constrains to initial primitives. Finally, we summarize the challenges, expected applications, and describe possible future for primitive extraction methods that can achieve globally optimal results efficiently, even with disorganized, uneven, noisy, incomplete, and large-scale lidar point clouds.

@article{
 title = {PrimiTect: Fast Continuous Hough Voting for Primitive Detection},
 type = {article},
 year = {2020},
 pages = {8404-8410},
 id = {1a5c069d-f0f3-3c09-9103-f8fe7865df16},
 created = {2021-01-25T14:53:33.924Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2021-02-09T17:05:55.079Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,beecb55d-84d0-48a2-a344-e50cfe559467},
 private_publication = {false},
 abstract = {This paper tackles the problem of data abstraction in the context of 3D point sets. Our method classifies points into different geometric primitives, such as planes and cones, leading to a compact representation of the data. Being based on a semi-global Hough voting scheme, the method does not need initialization and is robust, accurate, and efficient. We use a local, low-dimensional parameterization of primitives to determine type, shape and pose of the object that a point belongs to. This makes our algorithm suitable to run on devices with low computational power, as often required in robotics applications. The evaluation shows that our method outperforms state-of-the-art methods both in terms of accuracy and robustness.},
 bibtype = {article},
 author = {Sommer, Christiane and Sun, Yumin and Bylow, Erik and Cremers, Daniel},
 doi = {10.1109/ICRA40945.2020.9196988},
 journal = {Proceedings - IEEE International Conference on Robotics and Automation}
}

This paper tackles the problem of data abstraction in the context of 3D point sets. Our method classifies points into different geometric primitives, such as planes and cones, leading to a compact representation of the data. Being based on a semi-global Hough voting scheme, the method does not need initialization and is robust, accurate, and efficient. We use a local, low-dimensional parameterization of primitives to determine type, shape and pose of the object that a point belongs to. This makes our algorithm suitable to run on devices with low computational power, as often required in robotics applications. The evaluation shows that our method outperforms state-of-the-art methods both in terms of accuracy and robustness.

@article{
 title = {View planning in robot active vision: A survey of systems, algorithms, and applications},
 type = {article},
 year = {2020},
 keywords = {active vision,next-best view,robotic,sensor planning,view planning},
 pages = {225-245},
 volume = {6},
 id = {a9005365-b168-3b3c-8a9a-63b1f0f699bc},
 created = {2021-01-26T12:37:11.551Z},
 file_attached = {true},
 profile_id = {ad172e55-c0e8-3aa4-8465-09fac4d5f5c8},
 group_id = {1ff583c0-be37-34fa-9c04-73c69437d354},
 last_modified = {2023-08-08T11:39:19.280Z},
 read = {false},
 starred = {false},
 authored = {false},
 confirmed = {true},
 hidden = {false},
 citation_key = {Zeng2020a},
 folder_uuids = {c41ac501-6dd3-4d6f-b177-cdc2b43ddc1f,5439d198-93d5-4603-a7ce-201d423f231e,bc26f4dd-ccfc-4a52-b602-2ceb657d0906,13d43b82-d9b4-40a8-9031-8e926a718ef0,4f36a0a5-b08a-4f70-b020-4daf83cb0507},
 private_publication = {false},
 abstract = {Rapid development of artificial intelligence motivates researchers to expand the capabilities of intelligent and autonomous robots. In many robotic applications, robots are required to make planning decisions based on perceptual information to achieve diverse goals in an efficient and effective way. The planning problem has been investigated in active robot vision, in which a robot analyzes its environment and its own state in order to move sensors to obtain more useful information under certain constraints. View planning, which aims to find the best view sequence for a sensor, is one of the most challenging issues in active robot vision. The quality and efficiency of view planning are critical for many robot systems and are influenced by the nature of their tasks, hardware conditions, scanning states, and planning strategies. In this paper, we first summarize some basic concepts of active robot vision, and then review representative work on systems, algorithms and applications from four perspectives: object reconstruction, scene reconstruction, object recognition, and pose estimation. Finally, some potential directions are outlined for future work.},
 bibtype = {article},
 author = {Zeng, Rui and Wen, Yuhui and Zhao, Wang and Liu, Yong Jin},
 doi = {10.1007/s41095-020-0179-3},
 journal = {Computational Visual Media},
 number = {3}
}

Rapid development of artificial intelligence motivates researchers to expand the capabilities of intelligent and autonomous robots. In many robotic applications, robots are required to make planning decisions based on perceptual information to achieve diverse goals in an efficient and effective way. The planning problem has been investigated in active robot vision, in which a robot analyzes its environment and its own state in order to move sensors to obtain more useful information under certain constraints. View planning, which aims to find the best view sequence for a sensor, is one of the most challenging issues in active robot vision. The quality and efficiency of view planning are critical for many robot systems and are influenced by the nature of their tasks, hardware conditions, scanning states, and planning strategies. In this paper, we first summarize some basic concepts of active robot vision, and then review representative work on systems, algorithms and applications from four perspectives: object reconstruction, scene reconstruction, object recognition, and pose estimation. Finally, some potential directions are outlined for future work.