Allow: /robots.txt User-agent: Sosoimagespider+ Disallow: / User-agent: * Crawl-delay: 300 # Extended Standard Request-rate: 1/10 # maximum rate is one page every 10 seconds Visit-time: 1630-0000 # only visit between 06:00 and 08:45 UTC (GMT) #Allow: /portal.php$ Allow: /index.php$ #Allow: /*.html$ Disallow: /adm/ Disallow: /cache/ Disallow: /docs/ Disallow: /files/ Disallow: /images/ Disallow: /includes/ Disallow: /language/ Disallow: /g2/ Disallow: /cron.php* Disallow: /mcp.php* Disallow: /ucp.php* Disallow: /memberlist.php* Disallow: /posting.php* Disallow: /faq.php* Disallow: /gallery2.php* Disallow: /search.php