atridad/nginx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Found more crawlers

Browse Source
This commit is contained in:
Atridad Lahiji
2026-02-04 07:07:14 +00:00
parent c5df2b2815
commit 0a878c3c31
1 changed files with 46 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
fuckai/http_top.conf
View File

@@ -1,34 +1,66 @@
server_tokens off;
more_clear_headers Server;
more_clear_headers X-Powered-By;
client_body_buffer_size 10K;
client_header_buffer_size 1k;
large_client_header_buffers 2 1k;
map $http_user_agent $ai_crawler_block {
default 0;
"~*GPTBot" 1;
"~*ChatGPT-User" 1;
"~*OAI-SearchBot" 1;
"~*ChatGPT-Browser" 1;
"~*ClaudeBot" 1;
"~*Claude-Web" 1;
"~*anthropic-ai" 1;
"~*CCBot" 1;
"~*Anthropic-Claude" 1;
"~*xAI-Bot" 1;
"~*DeepseekBot" 1;
"~*Google-Extended" 1;
"~*Applebot-Extended" 1;
"~*Gemini-Ai" 1;
"~*Gemini-Deep-Research" 1;
"~*Google-CloudVertexBot" 1;
"~*Google-NotebookLM" 1;
"~*GoogleAgent-Mariner" 1;
"~*Bard-Ai" 1;
"~*FacebookBot" 1;
"~*Meta-ExternalAgent" 1;
"~*meta-webindexer" 1;
"~*Applebot-Extended" 1;
"~*bingbot" 1;
"~*CCBot" 1;
"~*PerplexityBot" 1;
"~*Perplexity-User" 1;
"~*Bytespider" 1;
"~*Diffbot" 1;
"~*Amazonbot" 1;
"~*cohere-ai" 1;
"~*Cohere-Command" 1;
"~*YouBot" 1;
"~*Omgilibot" 1;
"~*ImagesiftBot" 1;
"~*AI2Bot" 1;
"~*Andibot" 1;
"~*bigsur.ai" 1;
"~*Brightbot" 1;
"~*TerraCotta" 1;
"~*Character-AI" 1;
"~*Devin" 1;
"~*Crawlspace" 1;
"~*DuckAssistBot" 1;
"~*FirecrawlAgent" 1;
}
client_body_buffer_size 10K;
client_header_buffer_size 1k;
large_client_header_buffers 2 1k;
server_tokens off;
more_clear_headers Server;
more_clear_headers X-Powered-By;
"~*Groq-Bot" 1;
"~*HuggingFace-Bot" 1;
"~*IbouBot" 1;
"~*MistralAI-User" 1;
"~*Replicate-Bot" 1;
"~*RunPod-Bot" 1;
"~*TimpiBot" 1;
"~*Together-Bot" 1;
"~*Kangaroo Bot" 1;
"~*PanguBot" 1;
"~*Cotoyogi" 1;
"~*Webzio-Extended" 1;
}