# Disallow all crawlers access to certain pages. User-agent: * Disallow: /common/ Disallow: /CommonPages/ Disallow: /commonpages/ Disallow: /include/ Disallow: /PricingPopup.aspx Disallow: /conditions.aspx Disallow: /request-callback.aspx Disallow: /request-callbak.aspx Disallow: /request-index.aspx Disallow: /request-callbaknew.aspx Disallow: /request-callbaksign-in.aspx Disallow: /request-callbakthanks.aspx # Disallow Following crawlers access to the site User-agent: ia_archiver-web.archive.org Disallow: / User-agent: ia_archiver Disallow: / User-agent: ia_archiver-web.archive.org Disallow: /robots.txt User-agent: ia_archiver-web.archive.org Disallow: robots.txt User-agent: ia_archiver Disallow: /robots.txt User-agent: ia_archiver Disallow: robots.txt User-agent: AboutUsBot Disallow: /