@article{14488, abstract = {Portrait viewpoint and illumination editing is an important problem with several applications in VR/AR, movies, and photography. Comprehensive knowledge of geometry and illumination is critical for obtaining photorealistic results. Current methods are unable to explicitly model in 3D while handling both viewpoint and illumination editing from a single image. In this paper, we propose VoRF, a novel approach that can take even a single portrait image as input and relight human heads under novel illuminations that can be viewed from arbitrary viewpoints. VoRF represents a human head as a continuous volumetric field and learns a prior model of human heads using a coordinate-based MLP with individual latent spaces for identity and illumination. The prior model is learned in an auto-decoder manner over a diverse class of head shapes and appearances, allowing VoRF to generalize to novel test identities from a single input image. Additionally, VoRF has a reflectance MLP that uses the intermediate features of the prior model for rendering One-Light-at-A-Time (OLAT) images under novel views. We synthesize novel illuminations by combining these OLAT images with target environment maps. Qualitative and quantitative evaluations demonstrate the effectiveness of VoRF for relighting and novel view synthesis, even when applied to unseen subjects under uncontrolled illumination. This work is an extension of Rao et al. (VoRF: Volumetric Relightable Faces 2022). We provide extensive evaluation and ablative studies of our model and also provide an application, where any face can be relighted using textual input.}, author = {Rao, Pramod and Mallikarjun, B. R. and Fox, Gereon and Weyrich, Tim and Bickel, Bernd and Pfister, Hanspeter and Matusik, Wojciech and Zhan, Fangneng and Tewari, Ayush and Theobalt, Christian and Elgharib, Mohamed}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, pages = {1148--1166}, publisher = {Springer Nature}, title = {{A deeper analysis of volumetric relightiable faces}}, doi = {10.1007/s11263-023-01899-3}, volume = {132}, year = {2024}, } @article{6944, abstract = {We study the problem of automatically detecting if a given multi-class classifier operates outside of its specifications (out-of-specs), i.e. on input data from a different distribution than what it was trained for. This is an important problem to solve on the road towards creating reliable computer vision systems for real-world applications, because the quality of a classifier’s predictions cannot be guaranteed if it operates out-of-specs. Previously proposed methods for out-of-specs detection make decisions on the level of single inputs. This, however, is insufficient to achieve low false positive rate and high false negative rates at the same time. In this work, we describe a new procedure named KS(conf), based on statistical reasoning. Its main component is a classical Kolmogorov–Smirnov test that is applied to the set of predicted confidence values for batches of samples. Working with batches instead of single samples allows increasing the true positive rate without negatively affecting the false positive rate, thereby overcoming a crucial limitation of single sample tests. We show by extensive experiments using a variety of convolutional network architectures and datasets that KS(conf) reliably detects out-of-specs situations even under conditions where other tests fail. It furthermore has a number of properties that make it an excellent candidate for practical deployment: it is easy to implement, adds almost no overhead to the system, works with any classifier that outputs confidence scores, and requires no a priori knowledge about how the data distribution could change.}, author = {Sun, Rémy and Lampert, Christoph}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {4}, pages = {970--995}, publisher = {Springer Nature}, title = {{KS(conf): A light-weight test if a multiclass classifier operates outside of its specifications}}, doi = {10.1007/s11263-019-01232-x}, volume = {128}, year = {2020}, } @article{6952, abstract = {We present a unified framework tackling two problems: class-specific 3D reconstruction from a single image, and generation of new 3D shape samples. These tasks have received considerable attention recently; however, most existing approaches rely on 3D supervision, annotation of 2D images with keypoints or poses, and/or training with multiple views of each object instance. Our framework is very general: it can be trained in similar settings to existing approaches, while also supporting weaker supervision. Importantly, it can be trained purely from 2D images, without pose annotations, and with only a single view per instance. We employ meshes as an output representation, instead of voxels used in most prior work. This allows us to reason over lighting parameters and exploit shading information during training, which previous 2D-supervised methods cannot. Thus, our method can learn to generate and reconstruct concave object classes. We evaluate our approach in various settings, showing that: (i) it learns to disentangle shape from pose and lighting; (ii) using shading in the loss improves performance compared to just silhouettes; (iii) when using a standard single white light, our model outperforms state-of-the-art 2D-supervised methods, both with and without pose supervision, thanks to exploiting shading cues; (iv) performance improves further when using multiple coloured lights, even approaching that of state-of-the-art 3D-supervised methods; (v) shapes produced by our model capture smooth surfaces and fine details better than voxel-based approaches; and (vi) our approach supports concave classes such as bathtubs and sofas, which methods based on silhouettes cannot learn.}, author = {Henderson, Paul M and Ferrari, Vittorio}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, pages = {835--854}, publisher = {Springer Nature}, title = {{Learning single-image 3D reconstruction by generative modelling of shape, pose and shading}}, doi = {10.1007/s11263-019-01219-8}, volume = {128}, year = {2020}, } @article{18361, abstract = {3D models of humans are commonly used within computer graphics and vision, and so the ability to distinguish between body shapes is an important shape retrieval problem. We extend our recent paper which provided a benchmark for testing non-rigid 3D shape retrieval algorithms on 3D human models. This benchmark provided a far stricter challenge than previous shape benchmarks. We have added 145 new models for use as a separate training set, in order to standardise the training data used and provide a fairer comparison. We have also included experiments with the FAUST dataset of human scans. All participants of the previous benchmark study have taken part in the new tests reported here, many providing updated results using the new data. In addition, further participants have also taken part, and we provide extra analysis of the retrieval results. A total of 25 different shape retrieval methods are compared.}, author = {Pickup, D. and Sun, X. and Rosin, P. L. and Martin, R. R. and Cheng, Z. and Lian, Z. and Aono, M. and Hamza, A. Ben and Bronstein, Alexander and Bronstein, M. and Bu, S. and Castellani, U. and Cheng, S. and Garro, V. and Giachetti, A. and Godil, A. and Isaia, L. and Han, J. and Johan, H. and Lai, L. and Li, B. and Li, C. and Li, H. and Litman, R. and Liu, X. and Liu, Z. and Lu, Y. and Sun, L. and Tam, G. and Tatsuma, A. and Ye, J.}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {2}, pages = {169--193}, publisher = {Springer Nature}, title = {{Shape retrieval of non-rigid 3D human models}}, doi = {10.1007/s11263-016-0903-8}, volume = {120}, year = {2016}, } @article{18358, abstract = {In this paper, the problem of non-rigid shape recognition is studied from the perspective of metric geometry. In particular, we explore the applicability of diffusion distances within the Gromov-Hausdorff framework. While the traditionally used geodesic distance exploits the shortest path between points on the surface, the diffusion distance averages all paths connecting the points. The diffusion distance constitutes an intrinsic metric which is robust, in particular, to topological changes. Such changes in the form of shortcuts, holes, and missing data may be a result of natural non-rigid deformations as well as acquisition and representation noise due to inaccurate surface construction. The presentation of the proposed framework is complemented with examples demonstrating that in addition to the relatively low complexity involved in the computation of the diffusion distances between surface points, its recognition and matching performances favorably compare to the classical geodesic distances in the presence of topological changes between the non-rigid shapes.}, author = {Bronstein, Alexander and Bronstein, Michael M. and Kimmel, Ron and Mahmoudi, Mona and Sapiro, Guillermo}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {2-3}, pages = {266--286}, publisher = {Springer Nature}, title = {{A Gromov-Hausdorff framework with diffusion geometry for topologically-robust non-rigid shape matching}}, doi = {10.1007/s11263-009-0301-6}, volume = {89}, year = {2010}, } @article{18359, abstract = {Symmetry and self-similarity are the cornerstone of Nature, exhibiting themselves through the shapes of natural creations and ubiquitous laws of physics. Since many natural objects are symmetric, the absence of symmetry can often be an indication of some anomaly or abnormal behavior. Therefore, detection of asymmetries is important in numerous practical applications, including crystallography, medical imaging, and face recognition, to mention a few. Conversely, the assumption of underlying shape symmetry can facilitate solutions to many problems in shape reconstruction and analysis. Traditionally, symmetries are described as extrinsic geometric properties of the shape. While being adequate for rigid shapes, such a description is inappropriate for non-rigid ones: extrinsic symmetry can be broken as a result of shape deformations, while its intrinsic symmetry is preserved. In this paper, we present a generalization of symmetries for non-rigid shapes and a numerical framework for their analysis, addressing the problems of full and partial exact and approximate symmetry detection and classification.}, author = {Raviv, Dan and Bronstein, Alexander and Bronstein, Michael M. and Kimmel, Ron}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, pages = {18--39}, publisher = {Springer Nature}, title = {{Full and partial symmetries of non-rigid shapes}}, doi = {10.1007/s11263-010-0320-3}, volume = {89}, year = {2010}, } @article{18360, abstract = {Many manifold learning procedures try to embed a given feature data into a flat space of low dimensionality while preserving as much as possible the metric in the natural feature space. The embedding process usually relies on distances between neighboring features, mainly since distances between features that are far apart from each other often provide an unreliable estimation of the true distance on the feature manifold due to its non-convexity. Distortions resulting from using long geodesics indiscriminately lead to a known limitation of the Isomap algorithm when used to map non-convex manifolds. Presented is a framework for nonlinear dimensionality reduction that uses both local and global distances in order to learn the intrinsic geometry of flat manifolds with boundaries. The resulting algorithm filters out potentially problematic distances between distant feature points based on the properties of the geodesics connecting those points and their relative distance to the boundary of the feature manifold, thus avoiding an inherent limitation of the Isomap algorithm. Since the proposed algorithm matches non-local structures, it is robust to strong noise. We show experimental results demonstrating the advantages of the proposed approach over conventional dimensionality reduction techniques, both global and local in nature.}, author = {Rosman, Guy and Bronstein, Michael M. and Bronstein, Alexander and Kimmel, Ron}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, pages = {56--68}, publisher = {Springer Nature}, title = {{Nonlinear dimensionality reduction by topologically constrained isometric embedding}}, doi = {10.1007/s11263-010-0322-1}, volume = {89}, year = {2010}, } @article{18356, abstract = {Similarity is one of the most important abstract concepts in human perception of the world. In computer vision, numerous applications deal with comparing objects observed in a scene with some a priori known patterns. Often, it happens that while two objects are not similar, they have large similar parts, that is, they are partially similar. Here, we present a novel approach to quantify partial similarity using the notion of Pareto optimality. We exemplify our approach on the problems of recognizing non-rigid geometric objects, images, and analyzing text sequences.}, author = {Bronstein, Alexander and Bronstein, Michael M. and Bruckstein, Alfred M. and Kimmel, Ron}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {2}, pages = {163--183}, publisher = {Springer Nature}, title = {{Partial similarity of objects, or how to compare a centaur to a horse}}, doi = {10.1007/s11263-008-0147-3}, volume = {84}, year = {2009}, } @article{18357, abstract = {This paper explores the problem of similarity criteria between nonrigid shapes. Broadly speaking, such criteria are divided into intrinsic and extrinsic, the first referring to the metric structure of the object and the latter to how it is laid out in the Euclidean space. Both criteria have their advantages and disadvantages: extrinsic similarity is sensitive to nonrigid deformations, while intrinsic similarity is sensitive to topological noise. In this paper, we approach the problem from the perspective of metric geometry. We show that by unifying the extrinsic and intrinsic similarity criteria, it is possible to obtain a stronger topology-invariant similarity, suitable for comparing deformed shapes with different topology. We construct this new joint criterion as a tradeoff between the extrinsic and intrinsic similarity and use it as a set-valued distance. Numerical results demonstrate the efficiency of our approach in cases where using either extrinsic or intrinsic criteria alone would fail.}, author = {Bronstein, Alexander and Bronstein, Michael M. and Kimmel, Ron}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {3}, pages = {281--301}, publisher = {Springer Nature}, title = {{Topology-invariant similarity of nonrigid shapes}}, doi = {10.1007/s11263-008-0172-2}, volume = {81}, year = {2009}, } @article{18354, abstract = {An expression-invariant 3D face recognition approach is presented. Our basic assumption is that facial expressions can be modelled as isometries of the facial surface. This allows to construct expression-invariant representations of faces using the bending-invariant canonical forms approach. The result is an efficient and accurate face recognition algorithm, robust to facial expressions, that can distinguish between identical twins (the first two authors). We demonstrate a prototype system based on the proposed algorithm and compare its performance to classical face recognition methods. The numerical methods employed by our approach do not require the facial surface explicitly. The surface gradients field, or the surface metric, are sufficient for constructing the expression-invariant representation of any given face. It allows us to perform the 3D face recognition task while avoiding the surface reconstruction stage.}, author = {Bronstein, Alexander and Bronstein, Michael M. and Kimmel, Ron}, issn = {1573-1405}, journal = {International Journal of Computer Vision}, number = {1}, pages = {5--30}, publisher = {Springer Nature}, title = {{Three-dimensional face recognition}}, doi = {10.1007/s11263-005-1085-y}, volume = {64}, year = {2005}, }