User-agent: * # match all bots Disallow: /MCAPDirect # advisers crawler to omit Disallow: /App_Themes Disallow: /App_GlobalResources Disallow: /Documents Disallow: /Common Disallow: /bin Disallow: /web.config Disallow: /*.axd$