Support X_CG_APIKEY env variable

.github/workflows/scraper.yaml

@@ -15,6 +15,7 @@ jobs:
       - run: 'echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" > key.json'
       # Not the best way to handle these secrets lol
       - run: 'echo "GOOGLE_APPLICATION_CREDENTIALS=\"key.json\"" >> .env'
+      - run: 'echo "X_CG_APIKEY=${{ secrets.X_CG_APIKEY }}" >> .env'
       - run: 'echo "USE_PROXY=${{ secrets.USE_PROXY }}" >> .env'
       - run: 'echo "USE_PRIVATE_PROXY=${{ secrets.USE_PRIVATE_PROXY }}" >> .env'
       - run: 'echo "PRIVATE_PROXY_HOST=${{ secrets.PRIVATE_PROXY_HOST }}" >> .env'

go.work

@@ -1,7 +1,5 @@
 go 1.21
 
-toolchain go1.21.6
-
 use (
 	./comparer
 	./inserter

scraper/go.mod

@@ -2,8 +2,6 @@ module scraper
 
 go 1.21
 
-toolchain go1.21.6
-
 require (
 	cloud.google.com/go/storage v1.30.1
 	github.com/joho/godotenv v1.5.1

scraper/pkg/do-api-call.go

@@ -118,6 +118,11 @@ func GetAllProducts() (
 // Retrieve a valid X-CG-APIKey from the xframe.js script.
 // Not providing this header will result in a 401.
 func GetXCGAPIKey() (XCGAPIKey string, err error) {
+	// if the token is in the env variables, return it
+	if e := os.Getenv("X_CG_APIKEY"); e != "" {
+		return e, nil
+	}
+
 	var browser *rod.Browser
 	var l *launcher.Launcher
 	if os.Getenv("HEADLESS") == "false" {
@@ -195,7 +200,7 @@ func GetXCGAPIKey() (XCGAPIKey string, err error) {
 		}
 	}
 
-	return "", fmt.Errorf("No API key found after 90 seconds.")
+	return "", fmt.Errorf("no API key found after 90 seconds")
 }
 
 func GetAllProductsWithParams(