@inproceedings{0471736d7f5f4d5a911e17ad0e5e3012,
title = "What deliberately degrading search quality tells us about discount functions",
abstract = "Deliberate degradation of search results is a common tool in user experiments. We degrade high-quality search results by inserting non-relevant documents at different ranks. The effect of these manipulations, on a number of commonly-used metrics, is counter-intuitive: the discount functions implicit in P@k, MRR, NDCG, and others do not account for the true relationship between rank and value to the user. We propose an alternative, based on visibility data.",
keywords = "Metrics, Result set manipulation",
author = "Paul Thomas and Timothy Jones and David Hawking",
year = "2011",
doi = "10.1145/2009916.2010072",
language = "English",
isbn = "9781450309349",
series = "SIGIR'11 - Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval",
publisher = "Association for Computing Machinery (ACM)",
pages = "1107--1108",
booktitle = "SIGIR'11 - Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval",
address = "United States",
note = "34th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2011 ; Conference date: 24-07-2011 Through 28-07-2011",
}