Sitemap: https://www.thecanvasfactory.com.au/sitemap.xml User-agent: * # BLOCK debug/admin/backend Disallow: /*dispatch=debug* Disallow: /*?dispatch=debug Disallow: /app/ Disallow: /design/backend/ Disallow: /design/themes/shinetech/ Disallow: /store_closed.html # CanvasApp: files loaded by ajax Disallow: /Css/CanvasApp.css.min.css Disallow: /Css/CanvasApp.mobile.css.min.css Disallow: /scripts/Canvasplus/canvasapp.js Disallow: /scripts/Canvasplus/canvasapp.messenger.js # Blog exclusions Disallow: /blog/tag/ Disallow: /blog/post-formats/ # Custom disallows Disallow: /Home/ Disallow: /catalog/FotoliaCatImages Disallow: /Catalog/FotoliaSearch Disallow: /canvas-art-prints-detail/ Disallow: /cdn-cgi/l/email-protection # ALLOW all public assets & pages Allow: /.js$ Allow: /.css$ Allow: /images/ Allow: /products/ Allow: /categories/ Allow: /blog/ Allow: /pages/ # Googlebot (web & image) User-agent: Googlebot Allow: / User-agent: Googlebot-image Allow: / # Bingbot & MSN User-agent: Bingbot User-agent: MSNBotMedia Allow: / # Other crawlers User-agent: AdsBot-Google User-agent: DuckDuckBot User-agent: Yahoo! Slurp User-agent: Baiduspider User-agent: YandexBot User-agent: Applebot User-agent: Facebot User-agent: Twitterbot User-agent: Pinterestbot User-agent: LinkedInBot User-agent: Slackbot-LinkExpanding User-agent: WhatsApp User-agent: TelegramBot Allow: / # Optional throttle # Crawl-delay: 5