User-agent: ia_archiver
Disallow: /samples
User-agent: 008
Disallow: /
User-agent: BoardReader
Disallow: /
User-agent: MJ12bot
Disallow: /
user-agent: arquivo-web-crawler
disallow: / 
user-agent: Sogou web spider
disallow: / 
user-agent: trendictionbot
disallow: / 
user-agent: SemrushBot
disallow: / 
user-agent: Riddler
disallow: / 
user-agent: DotBot
disallow: / 
user-agent: GrapeshotCrawle
disallow: / 
user-agent: MauiBot
disallow: / 
user-agent: CCBot
disallow: / 
user-agent: BLEXBot
disallow: / 
user-agent: AhrefsBot
disallow: / 
User-agent: Mediapartners-Google
disallow: /cgi/chart.cgi
Crawl-delay: 5.0
User-agent: Slurp
Crawl-delay: 3.0
user-agent: YandexBot
Disallow:	/astro-databank
user-agent: bingbot
Crawl-delay: 3.0
User-agent: Baiduspider
User-agent: baiduspider
User-agent: Baiduspider+
Crawl-delay: 5.0
Disallow:	/swisseph
user-agent: FreeFind
disallow:	/*.pdf$
disallow:	/cgi/chart.cgi
user-agent: OpenindexSpider
disallow:	/*.pdf$
disallow:	/cgi/chart.cgi
Disallow:	/astro-databank/Category:
user-agent: Googlebot
Allow:	/*.pdf$
Crawl-delay: 2.0
User-agent: *
Allow: /
Disallow:	/apache
Disallow:	/d
Disallow:	/design
Disallow:	/ew97
Disallow:	/ftp/swisseph/ephe/ast
Disallow:	/ftp/swisseph/ephe/jplfiles
Disallow:	/ftp/swisseph/ephe/longfiles
Disallow:	/h/gb
Disallow:	/hfot
Disallow:	/htm
Disallow:	/katalog
Disallow:	/license
Disallow:	/pair
Disallow:	/sgif
Disallow:	/sstat
Disallow:	/stats
Disallow:	/swisseph/*/ae16_
Disallow:	/swisseph/*/d5
Disallow:	/swisseph/ae/m0
Disallow:	/swisseph/ae/0
Disallow:	/swisseph/ae/1[0-7]
Disallow:	/swisseph/ae/21
Disallow:	/swisseph/aeh
Disallow:	/tmpd
Disallow:	/webp
Disallow:	/hhag
Disallow:	/hhae
Disallow:	/cgi/forum.cgi
Disallow:	/yabbfiles
Disallow:	/cgi/sub.cgi
Disallow:	/cgi/bugrep.cgi
Disallow:	/cgi/ovi.cgi
Disallow:	/cgi/ade.cgi
Disallow:	/astrowiki/de/Benutzer
Disallow:	/astrowiki/de/Diskussion
Disallow:	/astrowiki/en/User
Disallow:	/astrowiki/en/Talk
Disallow:	/astro-databank/Private:
Disallow:	/wiki/*/index.php
disallow:	/*.pdf$
Sitemap: http://www.astro.com/sitemap_index.xml