diff --git a/machines/_common/disallowed-user-agents.nix b/machines/_common/disallowed-user-agents.nix new file mode 100644 index 0000000..353b8dc --- /dev/null +++ b/machines/_common/disallowed-user-agents.nix @@ -0,0 +1,80 @@ +[ + # https://git.madhouse-project.org/iocaine/nam-shub-of-enki/src/branch/main/org/module + # https://momenticmarketing.com/blog/ai-search-crawlers-bots + # https://darkvisitors.com/agents + # ironically, https://www.amazon.com/robots.txt + # and https://radar.cloudflare.com/en-us/bots/directory?category=AI_CRAWLER&kind=all + # https://donotsta.re/objects/4ff43b3e-2a56-48f1-870b-48353cd90801 + # > DON’T LET YOUR DREAMS STAY DREAMS! + + # "Nexus 5X Build/MMB29P" + # apparently that also blocks normal googlebot. oops! + "Google-Extended" "Google-CloudVertexBot" "Gemini-Deep-Research" + "GoogleOther" "Google-NotebookLM" "GoogleAgent" + "FacebookBot" "meta-externalagent" "meta-externalfetcher" + "facebookexternalhit" "facebookcatalog" + "anthropic" "Claude-Web" "Claude-SearchBot" "Claude-User" "ClaudeBot" + "OpenAI" "GPTBot" "ChatGPT-User" "OAI-SearchBot" "ChatGPT Agent" + "PerplexityBot" "Perplexity-User" + "MistralAI-User" + "amazon" "NovaAct" # "Amazonbot" "amazon-kendra" + "Applebot-Extended" + "DuckAssistBot" "Copilot" "BingBot" "LinkedInBot" + "Bytespider" + "PetalBot" "PanguBot" # these are from huawei + "omgili" "webzio" # "Webzio-Extended" + "Anchor Browser" "Awario" "LinerBot" "factset_spyderbot" "magpie-crawler" + "CCBot" "YouBot" "Diffbot" "cohere-ai" "Novellum" "EchoboxBot" "WARDBot" + "Sidetrade indexer bot" "TimpiBot" "semrush" "Scrapy" "Devin" + "babbar.tech" "barkrowler" "BLEXBot" "DotBot" "ICC-Crawler" "Cotoyogi" + "ahrefs" "DataForSeoBot" "ImagesiftBot" "EtaoSpider" "QualifiedBot" + "MJ12Bot" "dataforseo-bot" "bigsur.ai" "Datenbank Crawler" "netEstate" + "LivelapBot" "SummalyBot" + "Firefox/72.0" "Arc/" + "Kangaroo Bot" # fucking crazy lmao + "TurnitinBot" # >This robot collects content from the Internet for the sole purpose of helping educational institutions prevent plagiarism. + # ,--------------------------------------. + # / Expanse, a Palo Alto Networks company, \ + # | searches across the global IPv4 space | + # | multiple times per day to identify | + # | customers' presences on the Internet.| + # | If you would like to be excluded from our| + # | scans, please send IP addresses/domains | + # \ to: scaninfo@paloaltonetworks.com / + # `---__ ________-------' + # -- _------------- + # \/ + # :nerd: + "Palo Alto Networks" + + "RepoLookoutBot" "Censys" + "SearchExpress" + "leakix" "l9scan" + "scanner.ducks.party" # youre a fucking bozo + "Netcraft" + # epic fail + # Mozilla/5.0 \(Windows NT 10.0\; Win64\; x64\) AppleWebKit/537.36 \(KHTML, like Gecko\) Chrome/100.0.4896.60 Safari/537.36 + "\\(" "\\)" "\\;" + # booo booo get better useragents + "SonyEricsson" "Symbian" "J2ME" + "Blackberry" "PlayBook" "RIM Tablet OS" + # MOT-V9mm/00.62 UP.Browser/6.2.3.4.c.1.123 (GUI) MMP/2.0 + "MOT-" # maybe? + "Windows ME" "Windows XP" "Windows CE" "Win 9x" + "IRIX" + # FAST-WebCrawler/3.8 (crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler) + # dude i cant even go to your website + # this might not even exist anymore + "FAST-WebCrawler" + "PPC Mac OS X" + "OS/2" # really bro + # "Mozilla/2.0 (compatible; Ask Jeeves/Teoma)" + # are you for fucking real + "Ask Jeeves" + "Opera" # this is old opera, new opera is "OPR/version" + # ELinks/0.9.3 (textmode; Linux 2.6.9-kanotix-8 i686; 127x41) + # where the hell are you getting these useragents from, gnu.org?? + "Linux 2" + "Knoppix" # nope lmao + "Konqueror" +] diff --git a/machines/fovps/services/caddy/default.nix b/machines/fovps/services/caddy/default.nix index 0300b74..c33df1e 100644 --- a/machines/fovps/services/caddy/default.nix +++ b/machines/fovps/services/caddy/default.nix @@ -1,86 +1,7 @@ { pkgs, config, ... }: let - disallowedAgents = [ - # https://git.madhouse-project.org/iocaine/nam-shub-of-enki/src/branch/main/org/module - # https://momenticmarketing.com/blog/ai-search-crawlers-bots - # https://darkvisitors.com/agents - # ironically, https://www.amazon.com/robots.txt - # and https://radar.cloudflare.com/en-us/bots/directory?category=AI_CRAWLER&kind=all - # https://donotsta.re/objects/4ff43b3e-2a56-48f1-870b-48353cd90801 - # > DON’T LET YOUR DREAMS STAY DREAMS! - - # "Nexus 5X Build/MMB29P" - # apparently that also blocks normal googlebot. oops! - "Google-Extended" "Google-CloudVertexBot" "Gemini-Deep-Research" - "GoogleOther" "Google-NotebookLM" "GoogleAgent" - "FacebookBot" "meta-externalagent" "meta-externalfetcher" - "facebookexternalhit" "facebookcatalog" - "anthropic" "Claude-Web" "Claude-SearchBot" "Claude-User" "ClaudeBot" - "OpenAI" "GPTBot" "ChatGPT-User" "OAI-SearchBot" "ChatGPT Agent" - "PerplexityBot" "Perplexity-User" - "MistralAI-User" - "amazon" "NovaAct" # "Amazonbot" "amazon-kendra" - "Applebot-Extended" - "DuckAssistBot" "Copilot" "BingBot" "LinkedInBot" - "Bytespider" - "PetalBot" "PanguBot" # these are from huawei - "omgili" "webzio" # "Webzio-Extended" - "Anchor Browser" "Awario" "LinerBot" "factset_spyderbot" "magpie-crawler" - "CCBot" "YouBot" "Diffbot" "cohere-ai" "Novellum" "EchoboxBot" "WARDBot" - "Sidetrade indexer bot" "TimpiBot" "semrush" "Scrapy" "Devin" - "babbar.tech" "barkrowler" "BLEXBot" "DotBot" "ICC-Crawler" "Cotoyogi" - "ahrefs" "DataForSeoBot" "ImagesiftBot" "EtaoSpider" "QualifiedBot" - "MJ12Bot" "dataforseo-bot" "bigsur.ai" "Datenbank Crawler" "netEstate" - "LivelapBot" "SummalyBot" - "Firefox/72.0" "Arc/" - "Kangaroo Bot" # fucking crazy lmao - "TurnitinBot" # >This robot collects content from the Internet for the sole purpose of helping educational institutions prevent plagiarism. - # ,--------------------------------------. - # / Expanse, a Palo Alto Networks company, \ - # | searches across the global IPv4 space | - # | multiple times per day to identify | - # | customers' presences on the Internet.| - # | If you would like to be excluded from our| - # | scans, please send IP addresses/domains | - # \ to: scaninfo@paloaltonetworks.com / - # `---__ ________-------' - # -- _------------- - # \/ - # :nerd: - "Palo Alto Networks" - - "RepoLookoutBot" "Censys" - "SearchExpress" - "leakix" "l9scan" - "scanner.ducks.party" # youre a fucking bozo - "Netcraft" - # epic fail - # Mozilla/5.0 \(Windows NT 10.0\; Win64\; x64\) AppleWebKit/537.36 \(KHTML, like Gecko\) Chrome/100.0.4896.60 Safari/537.36 - "\\(" "\\)" "\\;" - # booo booo get better useragents - "SonyEricsson" "Symbian" "J2ME" - "Blackberry" "PlayBook" "RIM Tablet OS" - # MOT-V9mm/00.62 UP.Browser/6.2.3.4.c.1.123 (GUI) MMP/2.0 - "MOT-" # maybe? - "Windows ME" "Windows XP" "Windows CE" "Win 9x" - "IRIX" - # FAST-WebCrawler/3.8 (crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler) - # dude i cant even go to your website - # this might not even exist anymore - "FAST-WebCrawler" - "PPC Mac OS X" - "OS/2" # really bro - # "Mozilla/2.0 (compatible; Ask Jeeves/Teoma)" - # are you for fucking real - "Ask Jeeves" - "Opera" # this is old opera, new opera is "OPR/version" - # ELinks/0.9.3 (textmode; Linux 2.6.9-kanotix-8 i686; 127x41) - # where the hell are you getting these useragents from, gnu.org?? - "Linux 2" - "Knoppix" # nope lmao - "Konqueror" - ]; + disallowedAgents = import ../../../_common/disallowed-user-agents.nix; in { imports = [ ./website.nix