# If the Joomla site is installed within a folder
# eg www.example.com/joomla/ then the robots.txt file
# MUST be moved to the site root
# eg www.example.com/robots.txt
# AND the joomla folder name MUST be prefixed to all of the
# paths.
# eg the Disallow rule for the /administrator/ folder MUST
# be changed to read
# Disallow: /joomla/administrator/
#
# For more information about the robots.txt standard, see:
# https://www.robotstxt.org/orig.html

User-agent: *
Disallow: /web/administrator/
Disallow: /web/bin/
Disallow: /web/cache/
Disallow: /web/cli/
Disallow: /web/components/
Disallow: /web/includes/
Disallow: /web/installation/
Disallow: /web/language/
Disallow: /web/layouts/
Disallow: /web/libraries/
Disallow: /web/logs/
Disallow: /web/modules/
Disallow: /web/plugins/
Disallow: /web/tmp/

Disallow: /js/
Disallow: /css/

#Bots
User-agent: SemrushBot
Disallow: /
Disallow: /web
User-agent: AhrefsBot
Disallow: /
Disallow: /web
User-agent: BLEXBot
Disallow: /
Disallow: /web
User-agent: PetalBot
Disallow: /
Disallow: /web
User-agent: bingbot
Disallow: /
Disallow: /web
User-agent: DotBot
Disallow: /
Disallow: /web
User-agent: Bingbot
Disallow: /
Disallow: /web
User-agent: BLP_bbot/0.1
Disallow: /
Disallow: /web
User-agent: BLP_bbot
Disallow: /
Disallow: /web
User-agent: MegaIndex.ru/2.0
Disallow: /
Disallow: /web
User-agent: MegaIndex.ru
Disallow: /
Disallow: /web
User-agent: megaIndex.ru
Disallow: /
Disallow: /web
User-agent: Linguee Bot
Disallow: /
Disallow: /web
User-agent: MJ12bot
Disallow: /
Disallow: /web

User-agent: Linguee
Disallow: /
Disallow: /web
User-agent: dotbot
Disallow: /
Disallow: /web
User-agent: blp_bbot/0.1
Disallow: /
Disallow: /web
User-agent: ccbot
Disallow: /
Disallow: /web
User-agent: CRMNLCrawlAgent
Disallow: /
Disallow: /web
User-agent: Cliqzbot
Disallow: /
Disallow: /web
User-agent: ZoominfoBot
Disallow: /
Disallow: /web