# robots.txt for http://www.csun.edu/ # permitting most crawling and scraping User-agent: * Disallow: # banning Google's robot from /homepage/* User-agent: googlebot Disallow: /homepage/* # banning Google's robot from /calendar/* User-agent: googlebot Disallow: /calendar/* # banning Google's robot from /account/* User-agent: googlebot Disallow: /account/* # banning Google's robot from /portal/* User-agent: googlebot Disallow: /portal/* # banning Google's robot from /test/* User-agent: googlebot Disallow: /test/* # banning Google's robot from /ua/email/* User-agent: googlebot Disallow: /ua/email/* # banning Google's robot from /search/* User-agent: googlebot Disallow: /search/* # banning Google's robot from /outage/* User-agent: googlebot Disallow: /outage/* # banning Google's robot from /_archives/* User-agent: googlebot Disallow: /_archives/* # banning Google's robot from /ua/development/annualgiving/* User-agent: googlebot Disallow: /ua/development/annualgiving/* # banning Google's robot from /studentaffairs.csun.edu/support/attachments/71/transfer_students_fall_2010__upload.csv* User-agent: googlebot Disallow: /studentaffairs.csun.edu/support/attachments/71/transfer_students_fall_2010__upload.csv* # banning Google's image index robot User-agent: Googlebot-Image Disallow: / # banning Yahoo! Image Search robot User-agent: Yahoo-MMCrawler Disallow: / # banning picsearch robot User-agent: psbot Disallow: /