User-agent: *                # directed to all spiders

Disallow: /cgi-bin/
Disallow: /intranet/

# This file stops robots to search certain directories