user-agent: *
disallow: /lib/
disallow: /*.php$
disallow: /pkginfo/
disallow: /report/
disallow: /var/
disallow: /catalog/
disallow: /customer/
disallow: /sendfriend/
disallow: /review/
disallow: /*SID=
disallow: /*?

# Fighting the spam bots
disallow: /mageworx_searchsuiteautocomplete/
disallow: /mobilelogin/
disallow: /rest/V1/integration/admin/token

# Disable checkout & customer account
disallow: /checkout/
disallow: /onestepcheckout/
disallow: /customer/
disallow: /customer/account/
disallow: /customer/account/login/

# Disable Search pages
disallow: /catalogsearch/
disallow: /catalog/product_compare/
disallow: /catalog/category/view/
disallow: /catalog/product/view/

# Disable common folders
disallow: /app/
disallow: /bin/
disallow: /dev/
disallow: /lib/
disallow: /phpserver/

# Disable Tag & Review (Avoid duplicate content)

disallow: /tag/
disallow: /review/

# Common files
disallow: /composer.json
disallow: /composer.lock
disallow: /CONTRIBUTING.md
disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
disallow: /COPYING.txt
disallow: /Gruntfile.js
disallow: /LICENSE.txt
disallow: /LICENSE_AFL.txt
disallow: /nginx.conf.sample
disallow: /package.json
disallow: /php.ini.sample
disallow: /RELEASE_NOTES.txt

# Disable sorting (Avoid duplicate content)
disallow: /*?*product_list_mode=
disallow: /*?*product_list_order=
disallow: /*?*product_list_limit=
disallow: /*?*product_list_dir=

# Disable version control folders and others
disallow: /*.git
disallow: /*.CVS
disallow: /*.Zip$
disallow: /*.Svn$
disallow: /*.Idea$
disallow: /*.Sql$
disallow: /*.Tgz$

# disallow everything from these crawlers
user-agent: Python-urllib
user-agent: Sogou web spider
user-agent: SeznamBot
user-agent: AhrefsBot
user-agent: robot
user-agent: Baidu
user-agent: BLEXBot
user-agent: SemrushBot
user-agent: nbot
user-agent: Dalvik	
user-agent: MJ12bot
user-agent: CFNetwork
user-agent: TelegramBot
user-agent: Yandex ( catchall )
user-agent: Curl
user-agent: proximic
user-agent: MojeekBot
user-agent: crawl
user-agent: MegaIndex.ru
user-agent: Mail.RU Bot
user-agent: Baiduspider
user-agent: Discordbot
user-agent: YandexBot
user-agent: SurdotlyBot
user-agent: LinkedInBot
user-agent: Mediatoolkitbot
user-agent: discover
user-agent: DotBot
user-agent: oBot
user-agent: Screaming Frog SEO Spider
user-agent: Slackbot-LinkExpanding
user-agent: netEstate NE Crawler
user-agent: blog
user-agent: CCBot
user-agent: Applebot
disallow: /

# Sitemap files
sitemap: https://www.store4riders.com/pub/media/sitemap/sitemap.xml
sitemap: https://www.store4riders.com/pub/brand-page-sitemap.txt
sitemap: https://www.store4riders.com/blog_sitemap.xml