chore: block additional scraper patterns

cal-itp · Jul 18, 2023 · 61d8082 · 61d8082
1 parent 77dfe5e
commit 61d8082
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/appcontainer/nginx.conf b/appcontainer/nginx.conf
@@ -44,15 +44,15 @@ http {
 
     # 404 known scraping path targets
     # case-insensitive regex matches the given path fragment anywhere in the request path
-    location ~* /(\.?git|api|app|assets|ats|bootstrap|bower|cgi|content|credentials|docker|doc|env|example|swagger|web) {
+    location ~* /(\.?git|api|app|assets|ats|bootstrap|bower|cgi|content|cpanel|credentials|debug|docker|doc|env|example|robots|swagger|web|yq) {
         access_log off;
         log_not_found off;
         return 404;
     }
 
     # 404 known scraping file targets
     # case-insensitive regex matches the given file extension anywhere in the request path
-    location ~* /.*\.(asp|axd|cgi|com|env|json|php|xml|ya?ml) {
+    location ~* /.*\.(asp|axd|cgi|com|env|json|php|ping|xml|ya?ml) {
         access_log off;
         log_not_found off;
         return 404;

diff --git a/tests/cypress/specs/scrapers.cy.js b/tests/cypress/specs/scrapers.cy.js
@@ -1,5 +1,12 @@
-const endpoints = ["cgi", "eligibility/app", "sample/api"];
-const files = [".env", "wp-admin/login.php", "data.json", "secrets/prod.yaml"];
+const endpoints = ["cgi", "eligibility/app", "sample/api", "cpaneladmin/"];
+const files = [
+  ".env",
+  "wp-admin/login.php",
+  "data.json",
+  "secrets/prod.yaml",
+  "robots.txt",
+  "sp/startSLO.ping",
+];
 const targets = endpoints.concat(files);
 
 const visit = (partial_path) => {