User-agent: *
Disallow: /

User-agent: wget
Allow: /

User-agent: ia_archiver
Allow: /

User-Agent: W3C-checklink
Allow: /