# robots.txt file for robot scanning of website

User-agent: *
Disallow: /

# This says to apply these settings to ALL search engine spiders/crawlers
User-agent: Slurp
Crawl-delay: 5
Disallow: /*.swf$
Disallow: /*.jpg$
Disallow: /*.pdf$
Disallow: /*.gif$
#Disallow: /*?osCid

User-agent: Mediapartners-Google
Allow: /

User-agent: Twiceler www.cuill.com/robots.html
Disallow: /

User-agent: Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)
Disallow: /

User-agent: Twiceler www.cuill.com/twiceler/robot.html
Disallow: /

User-agent: Twiceler-0.9 http://www.cuill.com/twiceler/robot.html
Disallow: /

User-agent: cuill 
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: DotBot
Disallow: /

#User-agent: spider
#Disallow: /