# Robots.txt file
 
 # All robots will spider the domain
User-agent: *
Disallow: /css/
Disallow: /fonts/
Disallow: /images/
Disallow: /js/
Disallow: /lib/
Disallow: /panel/
Disallow: /media/
Disallow: /static/
Disallow: /error
Disallow: /forbidden

User-agent: Slurp
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: psbot
Disallow: /

User-agent: The Knowledge AI 
Disallow:/

#test