Skip to content

Commit

Permalink
removing client header based bot restrictions
Browse files Browse the repository at this point in the history
  • Loading branch information
@ctot-nondef
ctot-nondef committed Feb 7, 2024
1 parent 3a370bf commit 48a6e89
Showing 1 changed file with 2 additions and 89 deletions.
91 changes: 2 additions & 89 deletions local/config/vufind/httpd-vufind.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,101 +32,14 @@ AliasMatch ^/resources/themes/([0-9a-zA-Z-_]*)/js/(.*)$ /usr/local/vufind/themes
# Configuration for public cache (used for asset pipeline minification)
AliasMatch ^/resources/cache/(.*)$ /usr/local/vufind/local/cache/public/$1
<Directory /usr/local/vufind/local/cache/public/>
<If "%{HTTP_USER_AGENT} =~ /bytedance/">
Require all denied
</If>
<ElseIf "%{HTTP_USER_AGENT} =~ /ahrefs/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /semrush/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /bytespider/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /gptbot/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /petalbot/">
Require all denied
</ElseIf>
#weird chinese "Liebao Fast" browser that makes up 20% of our traffic
#https://useragents.io/uas/mozilla-5-0-linux-android-7-1-1-oppo-r9sk-build-nmf26f-wv-applewebkit-537-36-khtml-like-gecko-version-4-0-chrome-70-0-3538-80-mobile-safari-537-36-liebaofast-5-12-3_3f6d65959714a6e2fc1984f1bb8b8cdc
<ElseIf "%{HTTP_USER_AGENT} =~ /NMF26F/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /bingbot/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /yandex/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /GoogleOther/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} == 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /dataforseo/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /181121/">
Require all denied
</ElseIf>
<Else>
Require all granted
</Else>
Require all granted
AllowOverride All
</Directory>

# Configuration for general VuFind base:
Alias /resources /usr/local/vufind/public
<Directory /usr/local/vufind/public/>
<If "%{HTTP_USER_AGENT} =~ /bytedance/">
Require all denied
</If>
<ElseIf "%{HTTP_USER_AGENT} =~ /ahrefs/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /semrush/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /bytespider/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /gptbot/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /petalbot/">
Require all denied
</ElseIf>
#weird chinese "Liebao Fast" browser that makes up 20% of our traffic
#https://useragents.io/uas/mozilla-5-0-linux-android-7-1-1-oppo-r9sk-build-nmf26f-wv-applewebkit-537-36-khtml-like-gecko-version-4-0-chrome-70-0-3538-80-mobile-safari-537-36-liebaofast-5-12-3_3f6d65959714a6e2fc1984f1bb8b8cdc
<ElseIf "%{HTTP_USER_AGENT} =~ /NMF26F/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /bingbot/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /yandex/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /GoogleOther/">
Require all denied
</ElseIf>
# this spKecific user agent accounts for another 20% of our requests
<ElseIf "%{HTTP_USER_AGENT} == 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /dataforseo/">
Require all denied
</ElseIf>
<ElseIf "%{HTTP_USER_AGENT} =~ /181121/">
Require all denied
</ElseIf>
<Else>
Require all granted
</Else>
Require all granted
AllowOverride All

# If you wish to use the Shibboleth authentication, uncomment the following lines
Expand Down

0 comments on commit 48a6e89

Please sign in to comment.