# This is just for bots that actually obey robots.txt files.  We block bots
# in other ways as well.

User-agent: GPTBot
Disallow: /

User-agent: DAUM 
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: uipbot/1.0 (uipbot@semasio.net)
Disallow: /


User-Agent: The Knowledge AI
Disallow: /
User-Agent: VelenPublicWebCrawler
Disallow: /

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /


User-agent: FemtosearchBot
Disallow: /

User-agent: Mediapartners-Google
Allow: /
User-agent: Yahoo-MMCrawler
Disallow: /

User-agent: Timpibot/0.8
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 120

User-agent: SiteAuditBot
Crawl-delay: 120

User-agent: SemrushBot-BA
Crawl-delay: 120

User-agent: SiteAuditBot/0.97
Crawl-delay: 120

User-agent: SemrushBot-SI
Crawl-delay: 120




User-agent: LCC
Disallow: /

User-agent: ltx71 - (http://ltx71.com/)
Disallow: /

User-agent: Barkrowler/0.9
Disallow: /
    

User-agent: ptd-crawler
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: Covario-IDS
Disallow: / 

User-agent: BLEXBot
Disallow: /

User-agent: Covario-IDS/1.0
Disallow: / 

User-agent: CCBot
Disallow: /

User-agent: ia_archiver
Disallow: /

user-agent: AhrefsBot
disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: coccoc
disallow: /

User-agent: Mixnode
Disallow: / 


User-agent: Aboundexbot
Disallow: / 

User-agent: bingbot
Disallow: /search/

User-agent: NTENTbot
Disallow:/

User-agent: *
Disallow: /loadcontent/
Disallow: /show/
Disallow: /cgi-bin/ 
Disallow: /cgi-bin/admin/ 
Disallow: /php/test/
Disallow: /cooking/questions/
#Disallow: /search/search.php
#Disallow: /sponsored/

User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: msnbot
Disallow: /*login.html*$

User-agent: msnbot
Disallow: /*addNewLink*.*$

User-agent: bingbot
Disallow: /search/*.*$

#User-agent: SemrushBot
#Disallow: /


User-Agent: NetResearchServer
Disallow: /

User-agent: AboutUsBot  
Disallow: / 

User-agent: psbot
Disallow: /

User-agent: Zeusbot
Disallow: /

User-agent: MSIECrawler
Disallow: /


User-agent: archive.org_bot
Disallow: /

User-agent: Java 1.1
Disallow: /

User-agent: lwp-trivial
Disallow: /



User-agent: Python-urllib
Disallow: /

User-agent: Python-webchecker
Disallow: /
User-agent: TurnitinBot
Disallow: /



User-agent: Wget
Disallow: /

User-agent: Zeus Link Scout
Disallow: /

User-Agent: TurnitinBot/1.4 http://www.turnitin.com/robot/crawlerinfo.html
Disallow: /

User-Agent: SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html
Disallow: /

User-Agent: FlashGet
Disallow: /