Relevance assessment: are judges exchangeable and does it matter.

@inproceedings{bailey-2008-relevance, author = {Peter Bailey and Nick Craswell and Ian Soboroff and Paul Thomas and Arjen P. de Vries and Emine Yilmaz}, editor = {Sung-Hyon Myaeng and Douglas W. Oard and Fabrizio Sebastiani and Tat-Seng Chua and Mun-Kew Leong}, title = {{Relevance assessment: are judges exchangeable and does it matter}}, booktitle = {{Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2008, Singapore, July 20-24, 2008}}, pages = {667--674}, publisher = {ACM}, year = {2008}, url = {https://doi.org/10.1145/1390334.1390447}, doi = {https://doi.org/10.1145/1390334.1390447} }