# # robots.txt for http://www.firstuu.org # # This file is used to exclude directories from # indexing by automated "robots" (used by search # engines like Google, AltaVisat, Yahoo, etc.) # # 9/16/2003 # # Modified for FreeFind 09/17/2009 to disallow the html pages for php migration. # # # These rules apply to all robots # User-agent: * Disallow: /review Disallow: /htdig Disallow: /Images Disallow: /webalizer Disallow: /mp3s Disallow: /html Crawl-delay: 10