User-agent: * # match all bots
Disallow: / # keep them out
Allow: /index.htm
Allow: /favicon.ico
Request-rate: 1/5         # maximum rate is one page every 5 seconds