A large-scale study of robots.txt.

@inproceedings{sun-2007-largescale, author = {Yang Sun and Ziming Zhuang and C. Lee Giles}, editor = {Carey L. Williamson and Mary Ellen Zurko and Peter F. Patel-Schneider and Prashant J. Shenoy}, title = {{A large-scale study of robots.txt}}, booktitle = {{Proceedings of the 16th International Conference on World Wide Web, WWW 2007, Banff, Alberta, Canada, May 8-12, 2007}}, pages = {1123--1124}, publisher = {ACM}, year = {2007}, url = {https://doi.org/10.1145/1242572.1242726}, doi = {https://doi.org/10.1145/1242572.1242726} }