diff --git a/src/static/robots.txt b/src/static/robots.txt index 7cc6b88..042af6e 100644 --- a/src/static/robots.txt +++ b/src/static/robots.txt @@ -1,20 +1,28 @@ User-agent: * Allow: / -User-agent: GPTBot +# AI Data Scraper + +User-agent: anthropic-ai +Disallow: / + +User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / -User-agent: ChatGPT-User +User-agent: FacebookBot Disallow: / User-agent: Google-Extended Disallow: / -User-agent: Omgilibot +User-agent: GPTBot Disallow: / -User-agent: FacebookBot +User-agent: omgili +Disallow: / + +User-agent: Omgilibot Disallow: /