add Google special crawler provider.

jonhadfield · Jul 2, 2024 · f12dc7b · f12dc7b
1 parent 35ab782
commit f12dc7b
Show file tree

Hide file tree

Showing 7 changed files with 416 additions and 0 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -187,6 +187,7 @@ const (
 	defaultGCPOutputPriority          = 200
 	defaultGoogleOutputPriority       = 200
 	defaultGooglebotOutputPriority    = 190
+	defaultGoogleSCOutputPriority     = 190
 	defaultiCloudPROutputPriority     = 100
 	defaultIPAPIOutputPriority        = 90
 	defaultIPURLOutputPriority        = 20
@@ -395,6 +396,23 @@ func initProviderConfig(sess *session.Session, v *viper.Viper) {
 
 	sess.Providers.Googlebot.URL = v.GetString("providers.googlebot.url")
 
+	// GoogleSC
+	if v.IsSet("providers.googlesc.enabled") {
+		sess.Providers.GoogleSC.Enabled = ToPtr(v.GetBool("providers.googlesc.enabled"))
+	} else {
+		sess.Messages.Mu.Lock()
+		sess.Messages.Info = append(sess.Messages.Info, "GoogleSC provider not defined in config")
+		sess.Messages.Mu.Unlock()
+	}
+
+	if v.IsSet("providers.googlesc.output_priority") {
+		sess.Providers.GoogleSC.OutputPriority = ToPtr(v.GetInt32("providers.GoogleSC.output_priority"))
+	} else {
+		sess.Providers.GoogleSC.OutputPriority = ToPtr(int32(defaultGoogleSCOutputPriority))
+	}
+
+	sess.Providers.GoogleSC.URL = v.GetString("providers.googlesc.url")
+
 	// iCloud Private Relay
 	if v.IsSet("providers.icloudpr.enabled") {
 		sess.Providers.ICloudPR.Enabled = ToPtr(v.GetBool("providers.icloudpr.enabled"))

diff --git a/process/process.go b/process/process.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/jonhadfield/ipscout/providers/azurewaf"
+	"github.com/jonhadfield/ipscout/providers/googlesc"
 
 	"github.com/jonhadfield/ipscout/providers/bingbot"
 
@@ -74,6 +75,7 @@ func getEnabledProviderClients(sess session.Session) (map[string]providers.Provi
 		{Name: gcp.ProviderName, Enabled: sess.Providers.GCP.Enabled, APIKey: "", NewClient: gcp.NewProviderClient},
 		{Name: google.ProviderName, Enabled: sess.Providers.Google.Enabled, APIKey: "", NewClient: google.NewProviderClient},
 		{Name: googlebot.ProviderName, Enabled: sess.Providers.Googlebot.Enabled, APIKey: "", NewClient: googlebot.NewProviderClient},
+		{Name: googlesc.ProviderName, Enabled: sess.Providers.GoogleSC.Enabled, APIKey: "", NewClient: googlesc.NewProviderClient},
 		{Name: ipapi.ProviderName, Enabled: sess.Providers.IPAPI.Enabled, APIKey: "", NewClient: ipapi.NewProviderClient},
 		{Name: ipurl.ProviderName, Enabled: sess.Providers.IPURL.Enabled, APIKey: "", NewClient: ipurl.NewProviderClient},
 		{Name: icloudpr.ProviderName, Enabled: sess.Providers.ICloudPR.Enabled, APIKey: "", NewClient: icloudpr.NewProviderClient},