User-agent: * # Some crawlers support this. The value is the number of seconds between requests. Crawl-delay: 30 ####### extended standards ######## # I doubt any crawlers support these "extended standards" directives. # maximum rate is one page every 10 seconds Request-rate: 1/30 # only visit between ~11:00 (1500=15:00 UTC/GMT) and 16:00 (2000=20:00 UTC/GMT) #Visit-time: 1500-2000 Disallow:/c/ Disallow:/CFIDE/ Disallow:/cwig/ Disallow:/ICF-HQ/ Disallow:/Macintosh HD/ Disallow:/users/ Disallow:/accessibility/ Disallow:/admin/ Disallow:/ffostercaremonth-archives/ Disallow:/fostercaremofostercaremonth-archives/ Disallow:/fostercaremonfostercaremonth-archives/ Disallow:/fostercaremonth-archives/ Disallow:/fostercaremonth-old/ Disallow:/admin/ Disallow:/tasks/ Disallow:/requirements/ Disallow:/config/ Disallow:/assets/ Disallow:/cart/ Disallow:/cb/ Disallow:/cfc/ Disallow:/css/ Disallow:/ch_emails/ Disallow:/CWLibrarian/ Disallow:/cwLibrarian/ Disallow:/email_announce/ Disallow:/calendar/cbconference/admin/ Disallow:/images/ Disallow:/ims/ Disallow:/includes/ Disallow:/search/ Disallow:/sqlTest/ Disallow:/survey/ Disallow:/targetedsurvey/ Disallow:/styleGuide/ Disallow:/sharedcontent/ Disallow:/test/ Disallow:/verityk2spider/ Disallow:/management/workforce/compendium/viewLog.cfm Disallow:/management/workforce/compendium/_runverity.cfm Disallow:/management/workforce/compendium/tempAdmin/ Disallow:/*.css$ Disallow:/*.js$ Disallow:/*.exe$ Disallow:/*.sql$ Disallow:/*.cfc$ Disallow:/*.png$ Disallow:/*.jpg$ Disallow:/*.gif$ Disallow:/*.tif$ Disallow:/*.svn$ Disallow:/topics/adoption/nam-2014/ Disallow:/fostercaremonth-archives/ Disallow:/fostercaremonth-old/ Disallow:/adoption/ Disallow:/Fostercaremonth2015/