For now I asked chatgtp to help me to implement a simple return 403 on bot user agent. I looked into my logs and collected the bot names which I saw. I know it won't hold forever but for now it's quite nice, I just added this file to /etc/nginx/conf.d/block_bots.conf and it gets run before all the vhosts and rejects all bots. The rest just goes normally to the vhosts. This way I don't need to implement it in each vhost seperatelly.

➜ jeena@Abraham conf.d cat block_bots.conf

# /etc/nginx/conf.d/block_bots.conf

# 1️⃣ Map user agents to $bad_bot
map $http_user_agent $bad_bot {
    default 0;

    ~*SemrushBot                            1;
    ~*AhrefsBot                             1;
    ~*PetalBot                              1;
    ~*YisouSpider                           1;
    ~*Amazonbot                             1;
    ~*VelenPublicWebCrawler                 1;
    ~*DataForSeoBot                          1;
    ~*Expanse,\ a\ Palo\ Alto\ Networks\ company 1;
    ~*BacklinksExtendedBot                   1;
    ~*ClaudeBot                              1;
    ~*OAI-SearchBot                          1;
    ~*GPTBot                                 1;
    ~*meta-externalagent                     1;
}

# 2️⃣ Global default server to block bad bots
server {
    listen 80 default_server;
    listen [::]:80 default_server;
    listen 443 ssl default_server;
    listen [::]:443 ssl default_server;

    # dummy SSL cert for HTTPS
    ssl_certificate     /etc/ssl/certs/ssl-cert-snakeoil.pem;
    ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;

    # block bad bots
    if ($bad_bot) {
        return 403;
    }

    # close connection for anything else hitting default server
    return 444;
}