user-agent: SiteCheck-sitecrawl
allow: /Wraps
allow: /Intranet

user-agent: SiteimproveBot
allow: /Wraps
allow: /Intranet

user-agent: Amazonbot
disallow: /

user-agent: Bytespider
disallow: /

user-agent: SemrushBot
disallow: /

user-agent: *

# wrap09 controls folders
disallow: /*/controls/

# Duplicated objects which might be old or in the process of being edited
disallow: /copy_of_
disallow: /*/copy_of_

# Root sites prefixed with zzz are in the process of being retired
disallow: /zzz

# Various ways of designating development sites
disallow: /Dev
disallow: /*_DEV
disallow: /*_Dev
disallow: /*_dev
disallow: /*-DEV
disallow: /*-Dev
disallow: /*-dev

# Various ways of designating old content that we aren't ready to delete
disallow: /*_OLD
disallow: /*_Old
disallow: /*_old
disallow: /*-OLD
disallow: /*-Old
disallow: /*-old

# Various ways of designating archive folders of old content
disallow: /*/ARCHIVE
disallow: /*/Archive
disallow: /*/archive
disallow: /*_ARCHIVE
disallow: /*_Archive
disallow: /*_archive
disallow: /*-ARCHIVE
disallow: /*-Archive
disallow: /*-archive

# Specific sites and subfolders that should not be indexed
disallow: /Wraps
disallow: /Intranet
disallow: /Engr/Research/EURO/SURF/Students
disallow: /BME/Academics/Undergraduate/SeniorDesign/Projects
disallow: /Impulse/uploads  # Not sure if this still exists because it's not in Zope

Crawl-delay: 5  # Google ignores this, but some crawlers might pay attention