#Add Robots Exclusion Commands for www below this line
User-agent: SemrushBot
Disallow: /

User-agent: SemanticScholarBot
Disallow: /

User-agent: PetalBot
Disallow: /

#Baiduspider
User-agent: Baiduspider
Disallow: /

#Yandex
User-agent: Yandex
Disallow: /

User-Agent: trendictionbot
Disallow: /


User-agent: *
Disallow: /commencement/book/
Disallow: /academicaffairs/commencement/book/
Disallow: /cos/satoshi-takahashi
Disallow: /cos/old
Disallow: /directory*
Disallow: /ntid/educationalmaterials
Disallow: /study/former-ceramics-bfa
Disallow: /study/former-fine-arts-studio-bfa
Disallow: /study/former-furniture-design-bfa
Disallow: /study/former-glass-bfa
Disallow: /study/former-metals-and-jewelry-design-bfa
Disallow: /cla/modernlanguages/
Disallow: /blog/
Disallow: /its/old
Disallow: /its/new

#allow the RIT Stormcrawler and Google
User-agent: RIT Storm Crawler
User-agent: Googlebot
User-agent: google
Disallow: /ntid/educationalmaterials
Disallow: /controller/newsite

#allow the Siteimprove crawler
User-agent: SiteimproveBot
Disallow:
User-agent: SiteimproveBot-Crawler
Disallow:

User-agent: AHC/1.0
Disallow: /
User-agent: AHC/2.0
Disallow: /
User-agent: AHC/2.1
Disallow: /

#Removal of /~w-* URLS from search indexes
#We can't do this globally, since many sites are broken and use these URLs publically
Disallow: /~w-oce/
Disallow: /~pltw/
Disallow: /~w-cosold/