@inproceedings{f332b151599a415d9dc910df44ad6e77,
title = "Knowledge Guided Attention and Inference for Describing Images Containing Unseen Objects",
abstract = "Images on the Web encapsulate diverse knowledge about varied abstract concepts. They cannot be sufficiently described with models learned from image-caption pairs that mention only a small number of visual object categories. In contrast, large-scale knowledge graphs contain many more concepts that can be detected by image recognition models. Hence, to assist description generation for those images which contain visual objects unseen in image-caption pairs, we propose a two-step process by leveraging large-scale knowledge graphs. In the first step, a multi-entity recognition model is built to annotate images with concepts not mentioned in any caption. In the second step, those annotations are leveraged as external semantic attention and constrained inference in the image description generation model. Evaluations show that our models outperform most of the prior work on out-of-domain MSCOCO image description generation and also scales better to broad domains with more unseen objects.",
author = "Aditya Mogadala and Umanga Bista and Lexing Xie and Achim Rettinger",
note = "Publisher Copyright: {\textcopyright} 2018, Springer International Publishing AG, part of Springer Nature.; 15th International Conference on Extended Semantic Web Conference, ESWC 2018 ; Conference date: 03-06-2018 Through 07-06-2018",
year = "2018",
doi = "10.1007/978-3-319-93417-4_27",
language = "English",
isbn = "9783319934167",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Verlag",
pages = "415--429",
editor = "Aldo Gangemi and Rapha{\"e}l Troncy and Roberto Navigli and Laura Hollink and Maria-Esther Vidal and Pascal Hitzler and Anna Tordai and Mehwish Alam",
booktitle = "The Semantic Web - 15th International Conference, ESWC 2018, Proceedings",
address = "Germany",
}