# robots.txt for http://www.surrey.ac.uk/ # see for an explanation. User-agent: MOMspider # The Multi-Owner Maintenance Spider Disallow: */scripts/ # Script files Disallow: /cgi-bin/ # Script files Disallow: /Web_Admin/ # Local MOMspider output Disallow: /redesign/ # draft content - added by Matt Farrow 13 August 2007 Disallow: /postgraduate2008/ # Temporary home for draft PG08 prospectus - added by Matt Farrow 13 August 2007 User-agent: * # All other spiders should avoid Disallow: */scripts/ # Script files Disallow: /cgi-bin/ # Script files Disallow: /Web_Admin/ # Huge server statistic logs Disallow: /redesign/ # draft content - added by Matt Farrow 13 August 2007 Disallow: /postgraduate2008/ # Temporary home for draft PG08 prospectus - added by Matt Farrow 13 August 2007