A Large-Scale Study of Relevance Assessments with Large Language Models Using UMBRELA.

@inproceedings{upadhyay-2025-largescale, author = {Shivani Upadhyay and Ronak Pradeep and Nandan Thakur and Daniel Campos and Nick Craswell and Ian Soboroff and Jimmy Lin}, editor = {Hamed Zamani and Laura Dietz and Benjamin Piwowarski and Sebastian Bruch}, title = {{A Large-Scale Study of Relevance Assessments with Large Language Models Using UMBRELA}}, booktitle = {{Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval, ICTIR 2025, Padua, Italy, 18 July 2025}}, pages = {358--368}, publisher = {ACM}, year = {2025}, url = {https://doi.org/10.1145/3731120.3744605}, doi = {https://doi.org/10.1145/3731120.3744605} }