Origin/proxy v2

.air.toml

@@ -7,21 +7,21 @@ tmp_dir = "tmp"
   bin = "./tmp/main"
   cmd = "go build -o ./tmp/main ./cmd"
   delay = 1000
-  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
-  exclude_file = []
+  exclude_dir = ["assets", "tmp", "vendor", "testdata",]
+  exclude_file = ["proxychain/ruleset/rule_resmod_types.gen.go", "proxychain/ruleset/rule_reqmod_types.gen.go", "handlers/api_modifiers_structdef.gen.go"]
   exclude_regex = ["_test.go"]
   exclude_unchanged = false
   follow_symlink = false
-  full_bin = "RULESET=./ruleset.yaml ./tmp/main"
+  full_bin = "./tmp/main --ruleset ./rulesets_v2"
   include_dir = []
-  include_ext = ["go", "tpl", "tmpl", "yaml", "html"]
+  include_ext = ["go", "tpl", "tmpl", "yaml", "html", "js"]
   include_file = []
   kill_delay = "0s"
   log = "build-errors.log"
   poll = false
   poll_interval = 0
   post_cmd = []
-  pre_cmd = ["echo 'dev' > handlers/VERSION"]
+  pre_cmd = ["git submodule update --init --recursive; git rev-parse --short HEAD > handlers/VERSION; git rev-parse --short HEAD > cmd/VERSION; cd proxychain/codegen && go run codegen.go && cd ../../handlers/api_modifiers_codegen && go run api_modifiers_codegen.go"]
   rerun = false
   rerun_delay = 500
   send_interrupt = false

.github/workflows/build-css.yaml

@@ -3,7 +3,10 @@ name: Build Tailwind CSS
 on:
   push:
     paths:
+      - "handlers/error_page.html"
       - "handlers/form.html"
+      - "handlers/playground.html"
+      - "proxychain/responsemodifiers/vendor/generate_readable_outline.html"
   workflow_dispatch:
 
 jobs:
@@ -27,16 +30,17 @@ jobs:
         name: Build Tailwind CSS
         run: pnpm build
       -
-        name: Commit generated stylesheet
+        name: Commit generated stylesheet for handlers/styles.css
         run: |
-          if git diff --quiet cmd/styles.css; then
+          if git diff --quiet handlers/styles.css; then
             echo "No changes to commit."
             exit 0
           else
             echo "Changes detected, committing..."
             git config --global user.name "Github action"
             git config --global user.email "username@users.noreply.github.com"
-            git add cmd
+            git add handlers
+            git add proxychain/responsemodifiers/vendor/
             git commit -m "Generated stylesheet"
             git push
-          fi
+          fi

.gitignore

@@ -1,5 +1,8 @@
 # dev binary
 ladder
+tmp/main
+tmp
 
 VERSION
-output.css
+output.css
+.aider*

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "proxychain/responsemodifiers/vendor/ddg-tracker-surrogates"]
+	path = proxychain/responsemodifiers/vendor/ddg-tracker-surrogates
+	url = https://github.com/duckduckgo/tracker-surrogates
+[submodule "proxychain/requestmodifiers/vendor/ua-parser-js"]
+	path = proxychain/requestmodifiers/vendor/ua-parser-js
+	url = https://github.com/faisalman/ua-parser-js.git

Dockerfile

@@ -7,7 +7,7 @@ COPY . .
 
 RUN go mod download
 
-RUN CGO_ENABLED=0 GOOS=linux go build -o ladder cmd/main.go
+RUN make build
 
 FROM debian:12-slim as release
 
@@ -18,8 +18,4 @@ RUN chmod +x /app/ladder
 
 RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 
-#EXPOSE 8080
-
-#ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
 CMD ["sh", "-c", "/app/ladder"]

Makefile

@@ -1,3 +1,11 @@
+build:
+	cd proxychain/codegen && go run codegen.go
+	cd handlers/api_modifiers_codegen && go run api_modifiers_codegen.go
+	git submodule update --init --recursive
+	git rev-parse --short HEAD > handlers/VERSION
+	git rev-parse --short HEAD > cmd/VERSION
+	go build -o ladder -ldflags="-s -w" cmd/main.go
+
 lint:
 	gofumpt -l -w .
 	golangci-lint run -c .golangci-lint.yaml --fix
@@ -7,4 +15,7 @@ lint:
 
 install-linters:
 	go install mvdan.cc/gofumpt@latest
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2
+
+run:
+	go run ./cmd/.

README.md

@@ -91,18 +91,28 @@ Or create a bookmark with the following URL:
 ```javascript
 javascript:window.location.href="http://localhost:8080/"+location.href
 ```
+### Outline
+```bash
+curl -X GET "http://localhost:8080/outline/https://www.example.com"
+```
 
 ### API
 ```bash
-curl -X GET "http://localhost:8080/api/https://www.example.com"
+curl -X GET "http://localhost:8080/api/content/https://www.example.com"
 ```
 
 ### RAW
-http://localhost:8080/raw/https://www.example.com
+http://localhost:8080/api/raw/https://www.example.com
 
 
 ### Running Ruleset
-http://localhost:8080/ruleset
+http://localhost:8080/api/ruleset
+
+### Running Rule
+http://localhost:8080/api/ruleset/https://example.com
+
+### List available modifiers
+http://localhost:8080/api/modifiers
 
 ## Configuration
 
@@ -189,7 +199,10 @@ There is a basic ruleset available in a separate repository [ruleset.yaml](https
 To run a development server at http://localhost:8080:
 
 ```bash
-echo "dev" > handlers/VERSION
+git clone git@github.com-ladddder:everywall/ladder.git
+git submodule update --init --recursive
+echo "dev  " > handlers/VERSION
+echo "dev  " > cmd/VERSION
 RULESET="./ruleset.yaml" go run cmd/main.go
 ```
 

cmd/main.go

@@ -1,26 +1,23 @@
 package main
 
 import (
-	"embed"
+	_ "embed"
 	"fmt"
 	"log"
 	"os"
-	"strings"
 
-	"ladder/handlers"
-	"ladder/handlers/cli"
+	"github.com/everywall/ladder/handlers"
+	"github.com/everywall/ladder/internal/cli"
+
+	"github.com/everywall/ladder/proxychain/requestmodifiers/bot"
+	ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/fiber/v2/middleware/basicauth"
-	"github.com/gofiber/fiber/v2/middleware/favicon"
 )
 
-//go:embed favicon.ico
-var faviconData string
-
-//go:embed styles.css
-var cssData embed.FS
+//go:embed VERSION
+var version string
 
 func main() {
 	parser := argparse.NewParser("ladder", "Every Wall needs a Ladder")
@@ -41,6 +38,23 @@ func main() {
 		Help:     "This will spawn multiple processes listening",
 	})
 
+	verbose := parser.Flag("v", "verbose", &argparse.Options{
+		Required: false,
+		Help:     "Adds verbose logging",
+	})
+
+	randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{
+		Required: false,
+		Help:     "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request",
+	})
+
+	randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{
+		Required: false,
+		Help:     "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request",
+	})
+
+	// TODO: add version flag that reads from handers/VERSION
+
 	ruleset := parser.String("r", "ruleset", &argparse.Options{
 		Required: false,
 		Help:     "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.",
@@ -51,35 +65,46 @@ func main() {
 		Help:     "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.",
 	})
 
-	mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{
-		Required: false,
-		Help:     "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.",
-	})
-
 	mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{
 		Required: false,
-		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
+		Help:     "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.",
 	})
 
 	err := parser.Parse(os.Args)
 	if err != nil {
 		fmt.Print(parser.Usage(err))
 	}
 
+	if *randomGoogleBot {
+		err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+		if err != nil {
+			fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
+			fmt.Println("defaulting to known trusted Googlebot identity")
+		}
+	}
+
+	if *randomBingBot {
+		err := bot.BingBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json")
+		if err != nil {
+			fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error())
+			fmt.Println("defaulting to known trusted Bingbot identity")
+		}
+	}
+
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
-	if *mergeRulesets || *mergeRulesetsGzip {
+	if *mergeRulesets {
 		output := os.Stdout
 
 		if *mergeRulesetsOutput != "" {
 			output, err = os.Create(*mergeRulesetsOutput)
-			
+
 			if err != nil {
 				fmt.Println(err)
 				os.Exit(1)
 			}
 		}
 
-		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output)
+		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
@@ -91,28 +116,30 @@ func main() {
 		*prefork = true
 	}
 
+	var rs ruleset_v2.IRuleset
+
+	switch {
+	case *ruleset != "":
+		rs, err = ruleset_v2.NewRuleset(*ruleset)
+		if err != nil {
+			fmt.Printf("ERROR: failed to load ruleset from %s\n", *ruleset)
+		}
+	case os.Getenv("RULESET") != "":
+		rs = ruleset_v2.NewRulesetFromEnv()
+	}
+
 	app := fiber.New(
 		fiber.Config{
-			Prefork: *prefork,
-			GETOnly: true,
+			Prefork:               *prefork,
+			GETOnly:               false,
+			ReadBufferSize:        4096 * 4, // increase max header size
+			DisableStartupMessage: true,
 		},
 	)
 
-	userpass := os.Getenv("USERPASS")
-	if userpass != "" {
-		userpass := strings.Split(userpass, ":")
-
-		app.Use(basicauth.New(basicauth.Config{
-			Users: map[string]string{
-				userpass[0]: userpass[1],
-			},
-		}))
-	}
-
-	app.Use(favicon.New(favicon.Config{
-		Data: []byte(faviconData),
-		URL:  "/favicon.ico",
-	}))
+	app.Use(handlers.Auth())
+	app.Use(handlers.Favicon())
+	app.Use(handlers.RenderErrorPage())
 
 	if os.Getenv("NOLOGS") != "true" {
 		app.Use(func(c *fiber.Ctx) error {
@@ -122,23 +149,28 @@ func main() {
 		})
 	}
 
+	proxyOpts := &handlers.ProxyOptions{
+		Verbose: *verbose,
+		Ruleset: rs,
+	}
+
 	app.Get("/", handlers.Form)
 
-	app.Get("/styles.css", func(c *fiber.Ctx) error {
-		cssData, err := cssData.ReadFile("styles.css")
-		if err != nil {
-			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
-		}
+	app.Get("styles.css", handlers.Styles)
+	app.Get("script.js", handlers.Script)
+	app.Get("playground-script.js", handlers.Script)
 
-		c.Set("Content-Type", "text/css")
+	app.All("api/raw/*", handlers.NewRawProxySiteHandler(proxyOpts))
 
-		return c.Send(cssData)
-	})
+	app.Get("api/modifiers", handlers.NewAPIModifersListHandler(proxyOpts))
+	app.Get("api/ruleset/*", handlers.NewRulesetSiteHandler(proxyOpts))
+	app.Get("api/content/*", handlers.NewAPIContentHandler("api/outline/*", proxyOpts))
+
+	app.Get("outline/*", handlers.NewOutlineHandler("outline/*", proxyOpts))
+	app.All("playground/*", handlers.PlaygroundHandler("playground/*", proxyOpts))
 
-	app.Get("ruleset", handlers.Ruleset)
-	app.Get("raw/*", handlers.Raw)
-	app.Get("api/*", handlers.Api)
-	app.Get("/*", handlers.ProxySite(*ruleset))
+	app.All("/*", handlers.NewProxySiteHandler(proxyOpts))
 
+	fmt.Println(cli.StartupMessage(version, *port, *ruleset))
 	log.Fatal(app.Listen(":" + *port))
 }

docker-compose.yaml

@@ -3,7 +3,7 @@ services:
   ladder:
     image: ghcr.io/everywall/ladder:latest
     container_name: ladder
-    #build: .
+    build: .
     #restart: always
     #command: sh -c ./ladder
     environment: