[feature] Discover webfinger through host-meta (#1588)

* [feature] Discover webfinger through host-meta

This implements a fallback for discovering the webfinger endpoint in
case the /.well-known/webfinger endpoint wasn't properly redirected.
Some instances do this because the recommendation used to be to use
host-meta for the webfinger redirect in the before times.

Closes #1558.

* [bug] Ensure we only ever update cache on success

* [chore] Move finger tests to their own place

This adds a test suite for transport and moves the finger cache tests
into there instead of abusing the search test suite.

* [chore] cleanup the test a bit more

We don't really need a separate function for the oddly located webfinger
response as we check the full URL string anyway

* Address review comments

* [chore] update config example

* [chore] access DB only through state in controller

cmd/gotosocial/action/server/server.go

@@ -110,7 +110,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	oauthServer := oauth.New(ctx, dbService)
 	typeConverter := typeutils.NewConverter(dbService)
 	federatingDB := federatingdb.New(&state, typeConverter)
-	transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, client)
+	transportController := transport.NewController(&state, federatingDB, &federation.Clock{}, client)
 	federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
 
 	// decide whether to create a noop email sender (won't send emails) or a real one

example/config.yaml

@@ -287,6 +287,10 @@ cache:
     user-ttl: "5m"
     user-sweep-freq: "30s"
 
+    webfinger-max-size": 250
+    webfinger-ttl: "24h"
+    webfinger-sweep-freq": "15m"
+
 ######################
 ##### WEB CONFIG #####
 ######################

internal/api/model/well-known.go

@@ -18,6 +18,8 @@
 
 package model
 
+import "encoding/xml"
+
 // WellKnownResponse represents the response to either a webfinger request for an 'acct' resource, or a request to nodeinfo.
 // For example, it would be returned from https://example.org/.well-known/webfinger?resource=acct:some_username@example.org
 //
@@ -32,12 +34,12 @@ type WellKnownResponse struct {
 
 // Link represents one 'link' in a slice of links returned from a lookup request.
 //
-// See https://webfinger.net/
+// See https://webfinger.net/ and https://www.rfc-editor.org/rfc/rfc6415.html#section-3.1
 type Link struct {
-	Rel      string `json:"rel"`
-	Type     string `json:"type,omitempty"`
-	Href     string `json:"href,omitempty"`
-	Template string `json:"template,omitempty"`
+	Rel      string `json:"rel" xml:"rel,attr"`
+	Type     string `json:"type,omitempty" xml:"type,attr,omitempty"`
+	Href     string `json:"href,omitempty" xml:"href,attr,omitempty"`
+	Template string `json:"template,omitempty" xml:"template,attr,omitempty"`
 }
 
 // Nodeinfo represents a version 2.1 or version 2.0 nodeinfo schema.
@@ -87,3 +89,13 @@ type NodeInfoUsage struct {
 type NodeInfoUsers struct {
 	Total int `json:"total"`
 }
+
+// HostMeta represents a hostmeta document.
+// See: https://www.rfc-editor.org/rfc/rfc6415.html#section-3
+//
+// swagger:model hostmeta
+type HostMeta struct {
+	XMLName xml.Name `xml:"XRD"`
+	XMLNS   string   `xml:"xmlns,attr"`
+	Link    []Link   `xml:"Link"`
+}

internal/cache/gts.go

@@ -20,6 +20,7 @@ package cache
 
 import (
 	"codeberg.org/gruf/go-cache/v3/result"
+	"codeberg.org/gruf/go-cache/v3/ttl"
 	"github.com/superseriousbusiness/gotosocial/internal/cache/domain"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -71,6 +72,9 @@ type GTSCaches interface {
 
 	// User provides access to the gtsmodel User database cache.
 	User() *result.Cache[*gtsmodel.User]
+
+	// Webfinger
+	Webfinger() *ttl.Cache[string, string]
 }
 
 // NewGTS returns a new default implementation of GTSCaches.
@@ -91,6 +95,7 @@ type gtsCaches struct {
 	status        *result.Cache[*gtsmodel.Status]
 	tombstone     *result.Cache[*gtsmodel.Tombstone]
 	user          *result.Cache[*gtsmodel.User]
+	webfinger     *ttl.Cache[string, string]
 }
 
 func (c *gtsCaches) Init() {
@@ -106,6 +111,7 @@ func (c *gtsCaches) Init() {
 	c.initStatus()
 	c.initTombstone()
 	c.initUser()
+	c.initWebfinger()
 }
 
 func (c *gtsCaches) Start() {
@@ -145,6 +151,9 @@ func (c *gtsCaches) Start() {
 	tryUntil("starting gtsmodel.User cache", 5, func() bool {
 		return c.user.Start(config.GetCacheGTSUserSweepFreq())
 	})
+	tryUntil("starting gtsmodel.Webfinger cache", 5, func() bool {
+		return c.webfinger.Start(config.GetCacheGTSWebfingerSweepFreq())
+	})
 }
 
 func (c *gtsCaches) Stop() {
@@ -160,6 +169,7 @@ func (c *gtsCaches) Stop() {
 	tryUntil("stopping gtsmodel.Status cache", 5, c.status.Stop)
 	tryUntil("stopping gtsmodel.Tombstone cache", 5, c.tombstone.Stop)
 	tryUntil("stopping gtsmodel.User cache", 5, c.user.Stop)
+	tryUntil("stopping gtsmodel.Webfinger cache", 5, c.webfinger.Stop)
 }
 
 func (c *gtsCaches) Account() *result.Cache[*gtsmodel.Account] {
@@ -210,6 +220,10 @@ func (c *gtsCaches) User() *result.Cache[*gtsmodel.User] {
 	return c.user
 }
 
+func (c *gtsCaches) Webfinger() *ttl.Cache[string, string] {
+	return c.webfinger
+}
+
 func (c *gtsCaches) initAccount() {
 	c.account = result.New([]result.Lookup{
 		{Name: "ID"},
@@ -355,3 +369,10 @@ func (c *gtsCaches) initUser() {
 	}, config.GetCacheGTSUserMaxSize())
 	c.user.SetTTL(config.GetCacheGTSUserTTL(), true)
 }
+
+func (c *gtsCaches) initWebfinger() {
+	c.webfinger = ttl.New[string, string](
+		0,
+		config.GetCacheGTSWebfingerMaxSize(),
+		config.GetCacheGTSWebfingerTTL())
+}

internal/config/config.go

@@ -207,6 +207,10 @@ type GTSCacheConfiguration struct {
 	UserMaxSize   int           `name:"user-max-size"`
 	UserTTL       time.Duration `name:"user-ttl"`
 	UserSweepFreq time.Duration `name:"user-sweep-freq"`
+
+	WebfingerMaxSize   int           `name:"webfinger-max-size"`
+	WebfingerTTL       time.Duration `name:"webfinger-ttl"`
+	WebfingerSweepFreq time.Duration `name:"webfinger-sweep-freq"`
 }
 
 // MarshalMap will marshal current Configuration into a map structure (useful for JSON/TOML/YAML).

internal/config/defaults.go

@@ -166,6 +166,10 @@ var Defaults = Configuration{
 			UserMaxSize:   100,
 			UserTTL:       time.Minute * 5,
 			UserSweepFreq: time.Second * 30,
+
+			WebfingerMaxSize:   250,
+			WebfingerTTL:       time.Hour * 24,
+			WebfingerSweepFreq: time.Minute * 15,
 		},
 	},
 

internal/config/helpers.gen.go

@@ -3003,6 +3003,81 @@ func GetCacheGTSUserSweepFreq() time.Duration { return global.GetCacheGTSUserSwe
 // SetCacheGTSUserSweepFreq safely sets the value for global configuration 'Cache.GTS.UserSweepFreq' field
 func SetCacheGTSUserSweepFreq(v time.Duration) { global.SetCacheGTSUserSweepFreq(v) }
 
+// GetCacheGTSWebfingerMaxSize safely fetches the Configuration value for state's 'Cache.GTS.WebfingerMaxSize' field
+func (st *ConfigState) GetCacheGTSWebfingerMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.WebfingerMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSWebfingerMaxSize safely sets the Configuration value for state's 'Cache.GTS.WebfingerMaxSize' field
+func (st *ConfigState) SetCacheGTSWebfingerMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.WebfingerMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSWebfingerMaxSizeFlag returns the flag name for the 'Cache.GTS.WebfingerMaxSize' field
+func CacheGTSWebfingerMaxSizeFlag() string { return "cache-gts-webfinger-max-size" }
+
+// GetCacheGTSWebfingerMaxSize safely fetches the value for global configuration 'Cache.GTS.WebfingerMaxSize' field
+func GetCacheGTSWebfingerMaxSize() int { return global.GetCacheGTSWebfingerMaxSize() }
+
+// SetCacheGTSWebfingerMaxSize safely sets the value for global configuration 'Cache.GTS.WebfingerMaxSize' field
+func SetCacheGTSWebfingerMaxSize(v int) { global.SetCacheGTSWebfingerMaxSize(v) }
+
+// GetCacheGTSWebfingerTTL safely fetches the Configuration value for state's 'Cache.GTS.WebfingerTTL' field
+func (st *ConfigState) GetCacheGTSWebfingerTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.WebfingerTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSWebfingerTTL safely sets the Configuration value for state's 'Cache.GTS.WebfingerTTL' field
+func (st *ConfigState) SetCacheGTSWebfingerTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.WebfingerTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSWebfingerTTLFlag returns the flag name for the 'Cache.GTS.WebfingerTTL' field
+func CacheGTSWebfingerTTLFlag() string { return "cache-gts-webfinger-ttl" }
+
+// GetCacheGTSWebfingerTTL safely fetches the value for global configuration 'Cache.GTS.WebfingerTTL' field
+func GetCacheGTSWebfingerTTL() time.Duration { return global.GetCacheGTSWebfingerTTL() }
+
+// SetCacheGTSWebfingerTTL safely sets the value for global configuration 'Cache.GTS.WebfingerTTL' field
+func SetCacheGTSWebfingerTTL(v time.Duration) { global.SetCacheGTSWebfingerTTL(v) }
+
+// GetCacheGTSWebfingerSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.WebfingerSweepFreq' field
+func (st *ConfigState) GetCacheGTSWebfingerSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.WebfingerSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSWebfingerSweepFreq safely sets the Configuration value for state's 'Cache.GTS.WebfingerSweepFreq' field
+func (st *ConfigState) SetCacheGTSWebfingerSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.WebfingerSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSWebfingerSweepFreqFlag returns the flag name for the 'Cache.GTS.WebfingerSweepFreq' field
+func CacheGTSWebfingerSweepFreqFlag() string { return "cache-gts-webfinger-sweep-freq" }
+
+// GetCacheGTSWebfingerSweepFreq safely fetches the value for global configuration 'Cache.GTS.WebfingerSweepFreq' field
+func GetCacheGTSWebfingerSweepFreq() time.Duration { return global.GetCacheGTSWebfingerSweepFreq() }
+
+// SetCacheGTSWebfingerSweepFreq safely sets the value for global configuration 'Cache.GTS.WebfingerSweepFreq' field
+func SetCacheGTSWebfingerSweepFreq(v time.Duration) { global.SetCacheGTSWebfingerSweepFreq(v) }
+
 // GetAdminAccountUsername safely fetches the Configuration value for state's 'AdminAccountUsername' field
 func (st *ConfigState) GetAdminAccountUsername() (v string) {
 	st.mutex.Lock()

internal/transport/controller.go

@@ -32,9 +32,9 @@ import (
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/state"
 )
 
 // Controller generates transports for use in making federation requests to other servers.
@@ -47,7 +47,7 @@ type Controller interface {
 }
 
 type controller struct {
-	db        db.DB
+	state     *state.State
 	fedDB     federatingdb.DB
 	clock     pub.Clock
 	client    pub.HttpClient
@@ -57,14 +57,14 @@ type controller struct {
 }
 
 // NewController returns an implementation of the Controller interface for creating new transports
-func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
+func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
 	applicationName := config.GetApplicationName()
 	host := config.GetHost()
 	proto := config.GetProtocol()
 	version := config.GetSoftwareVersion()
 
 	c := &controller{
-		db:        db,
+		state:     state,
 		fedDB:     federatingDB,
 		clock:     clock,
 		client:    client,
@@ -138,7 +138,7 @@ func (c *controller) NewTransportForUsername(ctx context.Context, username strin
 		u = username
 	}
 
-	ourAccount, err := c.db.GetAccountByUsernameDomain(ctx, u, "")
+	ourAccount, err := c.state.DB.GetAccountByUsernameDomain(ctx, u, "")
 	if err != nil {
 		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
 	}