atridad/nginx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Found more crawlers

Browse Source
This commit is contained in:
Atridad Lahiji
2026-02-04 07:07:14 +00:00
parent c5df2b2815
commit 0a878c3c31
1 changed files with 46 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
fuckai/http_top.conf
View File

@@ -1,34 +1,66 @@
server_tokens off;
more_clear_headers Server;
more_clear_headers X-Powered-By;
client_body_buffer_size 10K;
client_header_buffer_size 1k;
large_client_header_buffers 2 1k;
map $http_user_agent $ai_crawler_block { map $http_user_agent $ai_crawler_block {
default 0; default 0;
"~*GPTBot" 1; "~*GPTBot" 1;
"~*ChatGPT-User" 1; "~*ChatGPT-User" 1;
"~*OAI-SearchBot" 1;
"~*ChatGPT-Browser" 1;
"~*ClaudeBot" 1; "~*ClaudeBot" 1;
"~*Claude-Web" 1; "~*Claude-Web" 1;
"~*anthropic-ai" 1; "~*anthropic-ai" 1;
"~*Anthropic-Claude" 1;
"~*CCBot" 1; "~*xAI-Bot" 1;
"~*DeepseekBot" 1;
"~*Google-Extended" 1; "~*Google-Extended" 1;
"~*Applebot-Extended" 1; "~*Gemini-Ai" 1;
"~*Gemini-Deep-Research" 1;
"~*Google-CloudVertexBot" 1;
"~*Google-NotebookLM" 1;
"~*GoogleAgent-Mariner" 1;
"~*Bard-Ai" 1;
"~*FacebookBot" 1; "~*FacebookBot" 1;
"~*Meta-ExternalAgent" 1; "~*Meta-ExternalAgent" 1;
"~*meta-webindexer" 1;
"~*Applebot-Extended" 1;
"~*bingbot" 1;
"~*CCBot" 1;
"~*PerplexityBot" 1; "~*PerplexityBot" 1;
"~*Perplexity-User" 1;
"~*Bytespider" 1; "~*Bytespider" 1;
"~*Diffbot" 1; "~*Diffbot" 1;
"~*Amazonbot" 1; "~*Amazonbot" 1;
"~*cohere-ai" 1; "~*cohere-ai" 1;
"~*Cohere-Command" 1;
"~*YouBot" 1; "~*YouBot" 1;
"~*Omgilibot" 1; "~*Omgilibot" 1;
"~*ImagesiftBot" 1; "~*ImagesiftBot" 1;
"~*AI2Bot" 1;
"~*Andibot" 1;
"~*bigsur.ai" 1;
"~*Brightbot" 1;
"~*TerraCotta" 1;
"~*Character-AI" 1;
"~*Devin" 1;
"~*Crawlspace" 1;
"~*DuckAssistBot" 1;
"~*FirecrawlAgent" 1; "~*FirecrawlAgent" 1;
} "~*Groq-Bot" 1;
"~*HuggingFace-Bot" 1;
client_body_buffer_size 10K; "~*IbouBot" 1;
client_header_buffer_size 1k; "~*MistralAI-User" 1;
large_client_header_buffers 2 1k; "~*Replicate-Bot" 1;
server_tokens off; "~*RunPod-Bot" 1;
more_clear_headers Server; "~*TimpiBot" 1;
more_clear_headers X-Powered-By; "~*Together-Bot" 1;
"~*Kangaroo Bot" 1;
"~*PanguBot" 1;
"~*Cotoyogi" 1;
"~*Webzio-Extended" 1;
}