Sitemap: http://www.nationaljournal.com/sitemap/sitemap_index.xml

User-agent: *
Disallow: /cgi-bin
Disallow: /almanac/2000/
Disallow: /almanac/2008/
Disallow: /almanac/2002/
Disallow: /almanac/2004/
Disallow: /almanac/2006/
Disallow: /almanac/1998/
Disallow: /almanac/area/
Disallow: /scripts
Disallow: /search
Disallow: /account
Disallow: /member
Disallow: /membership
Allow: /almanac/2010

User-agent: Yandex
Crawl-delay: 5  # Specifies a 5 second timeout

User-agent: ScoutJet
Crawl-delay: 5

User-agent: BLP_bbot/0.1
Disallow: /

# Exclude extensions
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.gif$
Disallow: /*.png$
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.mp3$
Disallow: /*.swf$
Disallow: /*.flv$
Disallow: /*.mp4$
Disallow: /*.avi$
Disallow: /*.txt$
Disallow: /*.ics$