# http{s,}://{,www.}crystalfaeries.net/robots.txt
# celeste:crystalfaery ROBOTS.TXT 2020-08-05 12:20:50+00:00
# E-mail celeste@crystalfaeries.net with constructive criticism.
#
# Accessing crystalfaeries.net with falsified agent identification
# or in violation of restrictions imposed by either:
# the site(s)-wide robots.txt file(s), or,
# the robots meta tags within (HTML, etc.) web pages,
# constitutes fraud which creates liabilities
# in the amount of 1 Oz. of silver per access.
#
# Additional terms and conditions applicable to access
# are hereby incorporated by reference to:
# http://crystalfaeries.net/terms.html
#
# Any change is effective immediately for new contracts,
# while old contracts expire 30 days after notice of change.
# If you do not agree to the terms and conditions, you are
# free to not access the contents.
#
# In any case, thou shalt not cause any copies of
# material from crystalfaeries.NET to get published on the web
# so that those copies appear in search engines.

# Site Map (for humans: http://crystalfaeries.net/sitemap.html) #

Sitemap: http://crystalfaeries.net/sitemap.gz

# Robots fully excluded #

User-agent: Auto Spider 1.0
Disallow: /
User-agent: Baiduspider/2.0
Disallow: /
User-agent: MSIE 9.0
Disallow: /
User-agent: Win32
Disallow: /

# Private User-level DownloadBots allowed for private use by individuals operating in private capacity: #

User-agent: curl
User-agent: HTTPie
User-agent: Python-urllib/2.7
User-agent: Wget
User-agent: htdig

Disallow: /audio/
Disallow: /boot/
Disallow: /.cache/
Disallow: /celeste/
Disallow: /cgi-bin/
Disallow: /contents.html
Disallow: /dev/
Disallow: /downloads/
Disallow: /etc/
Disallow: /fae/dosha.html
Disallow: /fay/
Disallow: /fey/
Disallow: /fey.crystalfaeries.net/
Disallow: /home/
Disallow: /imgs/
Disallow: /linkdoc/
Disallow: /lost+found/
Disallow: /media/
Disallow: /mnt/
Disallow: /opt/
Disallow: /posix/doc/jargon/
Disallow: /posix/share/ag/
Disallow: /proc/
Disallow: /rawdog.html
Disallow: /rawdog.rss
Disallow: /recently_linked.html
Disallow: /recently_updated.html
Disallow: /recent_youtubes.html
Disallow: /rescue/
Disallow: /root/
Disallow: /run/
Disallow: /servers.html
Disallow: /sitemap.html
Disallow: /src/
Disallow: /srv/
Disallow: /sys/
Disallow: /tmp/
Disallow: /usr/
Disallow: /var/
Disallow: /video/
Crawl-delay: 5

######################################################
# Other Robots partially excluded partially allowed: #
######################################################

User-agent: *

Disallow: /audio/
Disallow: /boot/
Disallow: /.cache/
Disallow: /celeste/
Disallow: /cgi-bin/
Disallow: /contents.html
Disallow: /dev/
Disallow: /documents/
Disallow: /downloads/
Disallow: /etc/
Disallow: /fae/dosha.html
Disallow: /fay/
Disallow: /fey/
Disallow: /fey.crystalfaeries.net/
Disallow: /home/
Disallow: /imgs/
Disallow: /linkdoc/
Disallow: /lost+found/
Disallow: /media/
Disallow: /mnt/
Disallow: /opt/
Disallow: /posix/doc/jargon/
Disallow: /posix/share/ag/
Disallow: /proc/
Disallow: /rawdog.html
Disallow: /rawdog.rss
Disallow: /recently_linked.html
Disallow: /recently_updated.html
Disallow: /recent_youtubes.html
Disallow: /rescue/
Disallow: /root/
Disallow: /run/
Disallow: /servers.html
Disallow: /sitemap.html
Disallow: /src/
Disallow: /srv/
Disallow: /sys/
Disallow: /tmp/
Disallow: /usr/
Disallow: /var/
Disallow: /video/
Crawl-delay: 5