User-agent: *
Disallow: /
User-agent: Googlebot
User-agent: soscan (+https://dataone.org/)
Allow: /$
Allow: /dataset.xhtml
Allow: /sitemap*
Allow: /api/datasets/:persistentId/thumbnail
#Allow: /javax.faces.resource/images/
Disallow: /dataverse/*?q
Disallow: /dataverse/*/search
Disallow: /dataset.xhtml?*&version=&q=
Disallow: /
Crawl-delay: 100
Sitemap: https://dataverse.harvard.edu/sitemap_index.xml
# Created initially using: http://www.mcanerin.com/EN/search-engine/robots-txt.asp
# Verified using: http://tool.motoricerca.info/robots-checker.phtml