#
# General rules for all the crawlers, except the user-agents specified further down.
#
# NOTE: Please update the other bot rules when adding a new allowance/disallowance,
#       as there is no inheritance mechanism between groups of user-agent in the robot.txt file.
#

User-agent: *
Crawl-delay: 30

Disallow: /customcf/
Noindex: /customcf/
Disallow: /download_brochure.cfm
Noindex: /download_brochure.cfm
Disallow: /js/
Noindex: /js/
Disallow: /loader.cfm
Noindex: /loader.cfm
Disallow: /myimd/
Noindex: /myimd/
Disallow: /networks/learningnetwork/wednesdaywebcasts/
Noindex: /networks/learningnetwork/wednesdaywebcasts/
Disallow: /style/
Noindex: /style/
Disallow: /ws/
Noindex: /ws/
Disallow: /xml/
Noindex: /xml/
Disallow: /renderhandlers/
Noindex: /renderhandlers/
Disallow: /Gateways/
Noindex: /Gateways/
Disallow: /Element/
Noindex: /Element/
Disallow: /Stylesamples/
Noindex: /Stylesamples/
Disallow: /CallToAction/
Noindex: /CallToAction/
Disallow: /*backup$
Noindex: /*backup$
Disallow: /*Backup$
Noindex: /*Backup$
Disallow: /Cms/
Noindex: /Cms/
Disallow: /owp/backup/
Noindex: /owp/backup/
Disallow: /*brochure.pdf$
Noindex: /*brochure.pdf$
Disallow: /*Brochure.pdf$
Noindex: /*Brochure.pdf$
Disallow: /*brochure*.pdf$
Noindex: /*brochure*.pdf$
Disallow: /*Brochure*.pdf$
Noindex: /*Brochure*.pdf$
Disallow: /calendar/
Noindex: /calendar/
Disallow: /events/
Noindex: /events/
Disallow: /event/
Noindex: /event/
Disallow: /uupload/
Noindex: /uupload/
Disallow: /news/wcc/
Noindex: /news/wcc/
Disallow: /news/webletter/
Noindex: /news/webletter/
Disallow: /news/medialetter/
Noindex: /news/medialetter/
Disallow: /networks/learningnetwork/portals/
Noindex: /networks/learningnetwork/portals/
Disallow: /ProgramCompare/
Noindex: /ProgramCompare/
Disallow: /ProgramRecommender/
Noindex: /ProgramRecommender/
Disallow: /commonspot/
Noindex: /commonspot/
Disallow: /news/index-search.cfm
Noindex: /news/index-search.cfm
Disallow: /networks/alumni/alumnidirectory/Alumni-Directory.cfm
Noindex: /networks/alumni/alumnidirectory/Alumni-Directory.cfm
Disallow: */challenges/archive.cfm
Noindex: */challenges/archive.cfm
Disallow: /*_migration/
Noindex: /*_migration/



#
# Specific rules for Twitter
#
User-agent: Twitterbot
# Allow Twitter is whitelisted as a social media sites to allow crawler to access page markup.
Allow: /event/
Disallow: /customcf/
Disallow: /download_brochure.cfm
Disallow: /js/
Disallow: /loader.cfm
Disallow: /myimd/
Disallow: /networks/learningnetwork/wednesdaywebcasts/
Disallow: /style/
Disallow: /ws/
Disallow: /xml/
Disallow: /renderhandlers/
Disallow: /Gateways/
Disallow: /Element/
Disallow: /Stylesamples/
Disallow: /CallToAction/
Disallow: /*backup$
Disallow: /*Backup$
Disallow: /Cms/
Disallow: /owp/backup/
Disallow: /*brochure.pdf$
Disallow: /*Brochure.pdf$
Disallow: /*brochure*.pdf$
Disallow: /*Brochure*.pdf$
Disallow: /calendar/
Disallow: /events/
Disallow: /uupload/
Disallow: /news/wcc/
Disallow: /news/webletter/
Disallow: /news/medialetter/
Disallow: /networks/learningnetwork/portals/
Disallow: /ProgramCompare/
Disallow: /ProgramRecommender/
Disallow: /commonspot/
Disallow: /news/index-search.cfm
Disallow: /networks/alumni/alumnidirectory/Alumni-Directory.cfm
Disallow: */challenges/archive.cfm
Disallow: /*_migration/



#
# Specific rules for Facebook
#
User-agent: facebookexternalhit
# Allow Twitter is whitelisted as a social media sites to allow crawler to access page markup.
Allow: /event/
Disallow: /customcf/
Disallow: /download_brochure.cfm
Disallow: /js/
Disallow: /loader.cfm
Disallow: /myimd/
Disallow: /networks/learningnetwork/wednesdaywebcasts/
Disallow: /style/
Disallow: /ws/
Disallow: /xml/
Disallow: /renderhandlers/
Disallow: /Gateways/
Disallow: /Element/
Disallow: /Stylesamples/
Disallow: /CallToAction/
Disallow: /*backup$
Disallow: /*Backup$
Disallow: /Cms/
Disallow: /owp/backup/
Disallow: /*brochure.pdf$
Disallow: /*Brochure.pdf$
Disallow: /*brochure*.pdf$
Disallow: /*Brochure*.pdf$
Disallow: /calendar/
Disallow: /events/
Disallow: /uupload/
Disallow: /news/wcc/
Disallow: /news/webletter/
Disallow: /news/medialetter/
Disallow: /networks/learningnetwork/portals/
Disallow: /ProgramCompare/
Disallow: /ProgramRecommender/
Disallow: /commonspot/
Disallow: /news/index-search.cfm
Disallow: /networks/alumni/alumnidirectory/Alumni-Directory.cfm
Disallow: */challenges/archive.cfm
Disallow: /*_migration/



#
# Specific rules for LinkedIn
#
User-agent: LinkedInBot
# Allow Twitter is whitelisted as a social media sites to allow crawler to access page markup.
Allow: /event/
Disallow: /customcf/
Disallow: /download_brochure.cfm
Disallow: /js/
Disallow: /loader.cfm
Disallow: /myimd/
Disallow: /networks/learningnetwork/wednesdaywebcasts/
Disallow: /style/
Disallow: /ws/
Disallow: /xml/
Disallow: /renderhandlers/
Disallow: /Gateways/
Disallow: /Element/
Disallow: /Stylesamples/
Disallow: /CallToAction/
Disallow: /*backup$
Disallow: /*Backup$
Disallow: /Cms/
Disallow: /owp/backup/
Disallow: /*brochure.pdf$
Disallow: /*Brochure.pdf$
Disallow: /*brochure*.pdf$
Disallow: /*Brochure*.pdf$
Disallow: /calendar/
Disallow: /events/
Disallow: /uupload/
Disallow: /news/wcc/
Disallow: /news/webletter/
Disallow: /news/medialetter/
Disallow: /networks/learningnetwork/portals/
Disallow: /ProgramCompare/
Disallow: /ProgramRecommender/
Disallow: /commonspot/
Disallow: /news/index-search.cfm
Disallow: /networks/alumni/alumnidirectory/Alumni-Directory.cfm
Disallow: */challenges/archive.cfm
Disallow: /*_migration/



#
# Sitemap files
# As non-group-member records, these are not tied to any specific user-agents and may be followed by all crawlers, provided it is not disallowed.
#
Sitemap: https://www.imd.org/sitemap_index.xml