# GlobalFinance.Net robots.txt file
# Settings for all bots

User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /tag/
Disallow: /author/
Disallow: /search/
Disallow: /comments/
Disallow: /category/
Disallow: /category/*/*
Disallow: /wget/
Disallow: /httpd/
Disallow: /i/
Disallow: /f/
Disallow: /t/
Disallow: /c/
Disallow: /j/


# The Google 

# Basic Googlebot stuff
User-agent: Googlebot
Disallow: /*/trackback
Disallow: /*/feed
Disallow: /comment/
Disallow: /*/comment
Disallow: /*/comments
Disallow: /*?*
Disallow: /*?
Disallow: /page/

# Mediapartners crawls pages to determine AdSense content
User-agent: Mediapartners-Google
Disallow: /

# Adsbot crawls pages to measure Adwords landing page quality 
User-agent: Adsbot-Google
Disallow: /

# Image crawls pages for image index  
User-agent: Googlebot-Image
Allow: /

# Mobile crawls pages for mobile index
User-agent: Googlebot-Mobile
Allow: /

# Other search agents
# Wayback machine 
User-agent: ia_archiver-web.archive.org
Disallow: /
User-agent:  ia_archiver
Disallow: /

# digg mirror
User-agent: duggmirror
Disallow: /

# Microsoft search
User-agent: MSNbot
Allow: /
 
# Our sitemap location

Sitemap: http://www.globalfinance.net/sitemap.xml