A larger scale study of robots.txt.

@inproceedings{kolay-2008-larger, author = {Santanu Kolay}, editor = {Jinpeng Huai and Robin Chen and Hsiao-Wuen Hon and Yunhao Liu and Wei-Ying Ma and Andrew Tomkins and Xiaodong Zhang}, title = {{A larger scale study of robots.txt}}, booktitle = {{Proceedings of the 17th International Conference on World Wide Web, WWW 2008, Beijing, China, April 21-25, 2008}}, pages = {1171--1172}, publisher = {ACM}, year = {2008}, url = {https://doi.org/10.1145/1367497.1367711}, doi = {https://doi.org/10.1145/1367497.1367711} }