[chore] consolidate caching libraries (#704)

* add miekg/dns dependency

* set/validate accountDomain

* move finger to dereferencer

* totally break GetRemoteAccount

* start reworking finger func a bit

* start reworking getRemoteAccount a bit

* move mention parts to namestring

* rework webfingerget

* use util function to extract webfinger parts

* use accountDomain

* rework finger again, final form

* just a real nasty commit, the worst

* remove refresh from account

* use new ASRepToAccount signature

* fix incorrect debug call

* fix for new getRemoteAccount

* rework GetRemoteAccount

* start updating tests to remove repetition

* break a lot of tests
Move shared test logic into the testrig,
rather than having it scattered all over
the place. This allows us to just mock
the transport controller once, and have
all tests use it (unless they need not to
for some other reason).

* fix up tests to use main mock httpclient

* webfinger only if necessary

* cheeky linting with the lads

* update mentionName regex
recognize instance accounts

* don't finger instance accounts

* test webfinger part extraction

* increase default worker count to 4 per cpu

* don't repeat regex parsing

* final search for discovered accountDomain

* be more permissive in namestring lookup

* add more extraction tests

* simplify GetParseMentionFunc

* skip long search if local account

* fix broken test

* consolidate to all use same caching libraries

Signed-off-by: kim <grufwub@gmail.com>

* perform more caching in the database layer

Signed-off-by: kim <grufwub@gmail.com>

* remove ASNote cache

Signed-off-by: kim <grufwub@gmail.com>

* update cache library, improve db tracing hooks

Signed-off-by: kim <grufwub@gmail.com>

* return ErrNoEntries if no account status IDs found, small formatting changes

Signed-off-by: kim <grufwub@gmail.com>

* fix tests, thanks tobi!

Signed-off-by: kim <grufwub@gmail.com>

Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

go.mod

@@ -3,14 +3,13 @@ module github.com/superseriousbusiness/gotosocial
 go 1.18
 
 require (
-	codeberg.org/gruf/go-byteutil v1.0.1
-	codeberg.org/gruf/go-cache/v2 v2.0.1
-	codeberg.org/gruf/go-debug v1.1.2
-	codeberg.org/gruf/go-errors/v2 v2.0.1
+	codeberg.org/gruf/go-byteutil v1.0.2
+	codeberg.org/gruf/go-cache/v2 v2.1.1
+	codeberg.org/gruf/go-debug v1.2.0
+	codeberg.org/gruf/go-errors/v2 v2.0.2
 	codeberg.org/gruf/go-mutexes v1.1.2
 	codeberg.org/gruf/go-runners v1.2.1
-	codeberg.org/gruf/go-store v1.3.7
-	github.com/ReneKroon/ttlcache v1.7.0
+	codeberg.org/gruf/go-store v1.3.8
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.1.0
 	github.com/gin-contrib/cors v1.3.1
@@ -45,6 +44,7 @@ require (
 	github.com/uptrace/bun/dialect/sqlitedialect v1.1.3
 	github.com/wagslane/go-password-validator v0.3.0
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
+	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
 	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
 	golang.org/x/text v0.3.7
@@ -54,12 +54,14 @@ require (
 )
 
 require (
-	codeberg.org/gruf/go-bitutil v1.0.0 // indirect
+	codeberg.org/gruf/go-atomics v1.1.0 // indirect
+	codeberg.org/gruf/go-bitutil v1.0.1 // indirect
 	codeberg.org/gruf/go-bytes v1.0.2 // indirect
 	codeberg.org/gruf/go-fastcopy v1.1.1 // indirect
 	codeberg.org/gruf/go-fastpath v1.0.3 // indirect
 	codeberg.org/gruf/go-hashenc v1.0.2 // indirect
 	codeberg.org/gruf/go-pools v1.1.0 // indirect
+	codeberg.org/gruf/go-sched v1.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b // indirect

go.sum

@@ -35,22 +35,25 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
-codeberg.org/gruf/go-bitutil v1.0.0 h1:pM3KeDACgQ0ARbdIVEuRyAi3XDyUQdoSR0X24nbCZt8=
+codeberg.org/gruf/go-atomics v1.1.0 h1:ni9QXYoRUFYQMXE3akWaUb1wMcPBDc05Md6Rgml7W58=
+codeberg.org/gruf/go-atomics v1.1.0/go.mod h1:a/4/y/LgvjxjQVnpoy1VVkOSzLS1W9i1g4SJ0nflAa4=
 codeberg.org/gruf/go-bitutil v1.0.0/go.mod h1:sb8IjlDnjVTz8zPK/8lmHesKxY0Yb3iqHWjUM/SkphA=
+codeberg.org/gruf/go-bitutil v1.0.1 h1:l8z9nOvCpHhicU2LZyJ6jLK03UNzCF6bxVCwu+VEenQ=
+codeberg.org/gruf/go-bitutil v1.0.1/go.mod h1:3ezHnADoiRJs9jgn65AEZ3HY7dsabAYLmmnIvseCGJI=
 codeberg.org/gruf/go-bytes v1.0.0/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg=
 codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo=
 codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg=
 codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
-codeberg.org/gruf/go-byteutil v1.0.1 h1:cOSaqe2aytOTAC5NM62LI0w8qPfJ9n2BBddk5KyMgd0=
-codeberg.org/gruf/go-byteutil v1.0.1/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
+codeberg.org/gruf/go-byteutil v1.0.2 h1:OesVyK5VKWeWdeDR00zRJ+Oy8hjXx1pBhn7WVvcZWVE=
+codeberg.org/gruf/go-byteutil v1.0.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
 codeberg.org/gruf/go-cache v1.1.2/go.mod h1:/Dbc+xU72Op3hMn6x2PXF3NE9uIDFeS+sXPF00hN/7o=
-codeberg.org/gruf/go-cache/v2 v2.0.1 h1:dyyfn6W6jfUlD/HWu5oz48sowSgsfKKeg2lU6T0gRww=
-codeberg.org/gruf/go-cache/v2 v2.0.1/go.mod h1:VyfrDnPVUXUKYVkXnFOHRO1EoN+8zrTC9jRU6VmL3p0=
-codeberg.org/gruf/go-debug v1.1.2 h1:7Tqkktg60M/4WtXTTNUFH2T/6irBw4tI4viv7IRLZDE=
-codeberg.org/gruf/go-debug v1.1.2/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg=
+codeberg.org/gruf/go-cache/v2 v2.1.1 h1:Ate0URlO6dVTYGxcIST9sDKtwOqrqBmlm73zy3Bq1+k=
+codeberg.org/gruf/go-cache/v2 v2.1.1/go.mod h1:XstSofaENTH3PNBoMrBcowFJ339OcJfTwO0PCOp5lKQ=
+codeberg.org/gruf/go-debug v1.2.0 h1:WBbTMnK1ArFKUmgv04aO2JiC/daTOB8zQGi521qb7OU=
+codeberg.org/gruf/go-debug v1.2.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg=
 codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=
-codeberg.org/gruf/go-errors/v2 v2.0.1 h1:Y2LNtwoXAk1LUcIPzbowXOi7YTLmzsHKPe2QRbMH1OU=
-codeberg.org/gruf/go-errors/v2 v2.0.1/go.mod h1:6sI75OmvXE2AtRm4WUyGMEyqEOKTsfe+CA+aBXwbtJY=
+codeberg.org/gruf/go-errors/v2 v2.0.2 h1:T9CqfC+ntSIQL5mdQxwHlUMod1htpgNe3P1tugxKlT4=
+codeberg.org/gruf/go-errors/v2 v2.0.2/go.mod h1:6sI75OmvXE2AtRm4WUyGMEyqEOKTsfe+CA+aBXwbtJY=
 codeberg.org/gruf/go-fastcopy v1.1.1 h1:HhPCeFdVR5pwiSVDnQEGJ+J2ny9b5QgfiESc0zrWQAY=
 codeberg.org/gruf/go-fastcopy v1.1.1/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
 codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
@@ -67,15 +70,15 @@ codeberg.org/gruf/go-pools v1.1.0/go.mod h1:ZMYpt/DjQWYC3zFD3T97QWSFKs62zAUGJ/tz
 codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
 codeberg.org/gruf/go-runners v1.2.1 h1:eXXofOMkLTkrdqI0qFYm00hP9IW42JZbc+IRKgVyQ28=
 codeberg.org/gruf/go-runners v1.2.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
-codeberg.org/gruf/go-store v1.3.7 h1:w1nUVXDpdgqXMBRA+8S9MautFhCJXU8H+lQasS5KYXw=
-codeberg.org/gruf/go-store v1.3.7/go.mod h1:vaWKWMUFgS7b4NcCo/vS+7K0uh9ZATOI9P60gRMKUSw=
+codeberg.org/gruf/go-sched v1.0.1 h1:+EAXSVI4orY5lMNX2Vrke/UxF2qjmEy6gHcyySZg/3k=
+codeberg.org/gruf/go-sched v1.0.1/go.mod h1:LFzosJL0yrCNtXg9Vq9iwr4q6ANuRirO2cVwKYH7CLs=
+codeberg.org/gruf/go-store v1.3.8 h1:7Hzzsa8gaOc6spuGWXJVUWRAyKiOR/m60/jNYrD8cT0=
+codeberg.org/gruf/go-store v1.3.8/go.mod h1:Fy5pXEHiIVFRWDx8DfILwXS1ulrj/jLdSK2C2oElz3I=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk=
-github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I=
 github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
@@ -238,7 +241,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -581,8 +584,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4=
-go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@@ -615,6 +616,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0=
+golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

internal/cache/account.go

@@ -1,120 +1,70 @@
 package cache
 
 import (
-	"sync"
+	"time"
 
-	"github.com/ReneKroon/ttlcache"
-	"github.com/sirupsen/logrus"
+	"codeberg.org/gruf/go-cache/v2"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// AccountCache is a wrapper around ttlcache.Cache to provide URL and URI lookups for gtsmodel.Account
+// AccountCache is a cache wrapper to provide URL and URI lookups for gtsmodel.Account
 type AccountCache struct {
-	cache *ttlcache.Cache   // map of IDs -> cached accounts
-	urls  map[string]string // map of account URLs -> IDs
-	uris  map[string]string // map of account URIs -> IDs
-	mutex sync.Mutex
+	cache cache.LookupCache[string, string, *gtsmodel.Account]
 }
 
 // NewAccountCache returns a new instantiated AccountCache object
 func NewAccountCache() *AccountCache {
-	c := AccountCache{
-		cache: ttlcache.NewCache(),
-		urls:  make(map[string]string, 100),
-		uris:  make(map[string]string, 100),
-		mutex: sync.Mutex{},
-	}
-
-	// Set callback to purge lookup maps on expiration
-	c.cache.SetExpirationCallback(func(key string, value interface{}) {
-		account, ok := value.(*gtsmodel.Account)
-		if !ok {
-			logrus.Panicf("AccountCache could not assert entry with key %s to *gtsmodel.Account", key)
-		}
-
-		c.mutex.Lock()
-		delete(c.urls, account.URL)
-		delete(c.uris, account.URI)
-		c.mutex.Unlock()
+	c := &AccountCache{}
+	c.cache = cache.NewLookup(cache.LookupCfg[string, string, *gtsmodel.Account]{
+		RegisterLookups: func(lm *cache.LookupMap[string, string]) {
+			lm.RegisterLookup("uri")
+			lm.RegisterLookup("url")
+		},
+
+		AddLookups: func(lm *cache.LookupMap[string, string], acc *gtsmodel.Account) {
+			if uri := acc.URI; uri != "" {
+				lm.Set("uri", uri, acc.ID)
+			}
+			if url := acc.URL; url != "" {
+				lm.Set("url", url, acc.ID)
+			}
+		},
+
+		DeleteLookups: func(lm *cache.LookupMap[string, string], acc *gtsmodel.Account) {
+			if uri := acc.URI; uri != "" {
+				lm.Delete("uri", uri)
+			}
+			if url := acc.URL; url != "" {
+				lm.Delete("url", url)
+			}
+		},
 	})
-
-	return &c
+	c.cache.SetTTL(time.Minute*5, false)
+	c.cache.Start(time.Second * 10)
+	return c
 }
 
 // GetByID attempts to fetch a account from the cache by its ID, you will receive a copy for thread-safety
 func (c *AccountCache) GetByID(id string) (*gtsmodel.Account, bool) {
-	c.mutex.Lock()
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
+	return c.cache.Get(id)
 }
 
 // GetByURL attempts to fetch a account from the cache by its URL, you will receive a copy for thread-safety
 func (c *AccountCache) GetByURL(url string) (*gtsmodel.Account, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.urls[url]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt account lookup
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
+	return c.cache.GetBy("url", url)
 }
 
 // GetByURI attempts to fetch a account from the cache by its URI, you will receive a copy for thread-safety
 func (c *AccountCache) GetByURI(uri string) (*gtsmodel.Account, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.uris[uri]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt account lookup
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
-}
-
-// getByID performs an unsafe (no mutex locks) lookup of account by ID, returning a copy of account in cache
-func (c *AccountCache) getByID(id string) (*gtsmodel.Account, bool) {
-	v, ok := c.cache.Get(id)
-	if !ok {
-		return nil, false
-	}
-
-	a, ok := v.(*gtsmodel.Account)
-	if !ok {
-		panic("account cache entry was not an account")
-	}
-
-	return copyAccount(a), true
+	return c.cache.GetBy("uri", uri)
 }
 
 // Put places a account in the cache, ensuring that the object place is a copy for thread-safety
 func (c *AccountCache) Put(account *gtsmodel.Account) {
 	if account == nil || account.ID == "" {
 		panic("invalid account")
 	}
-
-	c.mutex.Lock()
 	c.cache.Set(account.ID, copyAccount(account))
-	if account.URL != "" {
-		c.urls[account.URL] = account.ID
-	}
-	if account.URI != "" {
-		c.uris[account.URI] = account.ID
-	}
-	c.mutex.Unlock()
 }
 
 // copyAccount performs a surface-level copy of account, only keeping attached IDs intact, not the objects.