# robots.txt for NYP ORG 2016/09/12
User-agent: *
# Disallow: /.*/library
# Disallow: /cs/Satellite?c*
# Disallow: /cs/Satellite?pagename*
Allow: /cs/Satellite?pagename=NYP/Data/NYP_Multimedia_List&c=NYP_YouTube
Allow: /cs/Satellite?pagename=NYP/Data/NYP_Multimedia_List&c=NYP_Vimeo
Allow: /cs/Satellite?gsa=a&pagename=NYP/Data/NYP_Search_Result*
#Allow: http://www.nyp.org/cs/Satellite?pagename=NYP%2FData%2FNYP_Multimedia_List&c=NYP_YouTube
#Allow: http://www.nyp.org/cs/Satellite?pagename=NYP/Data/NYP_Multimedia_List&c=NYP_Vimeo
#Allow: http://www.nyp.org/cs/Satellite?gsa=a&pagename=NYP/Data/NYP_Search_Result*

# Added for ticket: 154204 
#Disallow: http://www.nyp.org/cs/Satellite?c=Page&childpagename=NYP%2FNYP_Layout&cid=1429302241696&pagename=NYP_Wrapper&rendermode=2015
#Disallow: /cs/Satellite?c=Page&childpagename=NYP%2FNYP_Layout&cid=1429302241696&pagename=NYP_Wrapper&rendermode=2015
# Commented the above lines and added the below lines for the email from support team Rajesh on Feb 25, 2020
Disallow: http://www.nyp.org/cs/Satellite?c=Page&childpagename=NYP%2FNYP_Layout&cid=
Disallow: /cs/Satellite?c=Page&childpagename=NYP%2FNYP_Layout&cid=
# End of block added for ticket: 154204 

Crawl-delay: 10
Sitemap: https://www.nyp.org/Sitemap-NYP.xml

User-agent: 360Spider
Disallow: /

User-agent: HTTPFetch Spider
Disallow: /

User-agent: discobot
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: Sosospider+
Disallow: /

User-agent: YodaoBot
Disallow: /

User-agent: JikeSpider
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: 008
Disallow: /

User-agent: Fess
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: SkimBot
Disallow: /

User-agent: GlobalSpec Link Checker
Disallow: /

User-agent: HiScan
Disallow: /

User-agent: daumoa
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: CovarioCSE
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: ZumBot
Disallow: /

User-agent: AppCodesCrawler
Disallow: /

User-agent: AppCodes
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: ScreenerBot
Disallow: /

User-agent: Slurp
Crawl-delay: 10

User-agent: bingbot
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10

User-agent: msnbot-media
Crawl-delay: 10

User-agent: Baiduspider
Crawl-delay: 10

User-agent: heritrix
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10