diff --git a/static/robots.txt b/static/robots.txt index eb9358c..c8a1b5e 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,20 +1,28 @@ User-Agent: * Disallow: /admin/ -User-agent: GPTBot +# AI Data Scraper + +User-agent: anthropic-ai +Disallow: / + +User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / -User-agent: ChatGPT-User +User-agent: FacebookBot Disallow: / User-agent: Google-Extended Disallow: / -User-agent: Omgilibot +User-agent: GPTBot Disallow: / -User-agent: FacebookBot +User-agent: omgili +Disallow: / + +User-agent: Omgilibot Disallow: /