When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment.

@inproceedings{yu-2026-llm, author = {Chuting Yu and Peter Bruza and Chuting Yu and Peter Bruza and Hang Li and Tie-Yan Liu and Guido Zuccon and Joel Mackenzie and Laurianne Sitbon and Joel Mackenzie and Laurianne Sitbon and Michael Lawley and Teerapong Leelanupab and Michael Lawley and Teerapong Leelanupab}, editor = {Hannah Bast and Alistair Moffat and Falk Scholer and Marc Najork and Min Zhang and Hannah Bast and Marc Najork and Min Zhang}, title = {{When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment}}, booktitle = {{Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2026, MelbourneVICAustralia, July 20-24, 2026}}, pages = {4322--4327}, publisher = {ACM}, year = {2026}, url = {https://doi.org/10.1145/3805712.3809905}, doi = {https://doi.org/10.1145/3805712.3809905} }