# Google Image
User-agent: Googlebot-Image
Disallow:
Allow: /*

# Google AdSense
User-agent: Mediapartners-Google*
Disallow:

# digg mirror
User-agent: duggmirror
Disallow: /

# Twiiter
User-agent: Twitterbot
Disallow:
Allow: /*

# Google News
User-agent: Googlebot-News
Disallow: /spotlight/
Disallow: /announcements/

# global
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /rss-feed/
Disallow: /feed/
Disallow: /comments/
Disallow: /cities/
Disallow: /types/
Disallow: /anniversaries/
Disallow: /companies/
Disallow: /test-hp/
Disallow: *preview=true*
Disallow: /?p=*
Disallow: *3933714/TOI*

Sitemap: http://www.timesofisrael.com/sitemap_index.xml