Skip to content

Commit

Permalink
rewrite caching
Browse files Browse the repository at this point in the history
  • Loading branch information
benleb committed Aug 13, 2022
1 parent 210a78e commit 04ad063
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 223 deletions.
19 changes: 10 additions & 9 deletions cmd/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,23 +205,24 @@ func formatEvent(ctx context.Context, g *gocui.Gui, event *collections.Event, no

var ensName string

cache := cache.New(ctx)

if viper.GetBool("redis.enabled") {
if rdb := cache.GetRedisClient(); rdb != nil {
if cachedName, err := rdb.Get(context.Background(), cache.KeyENS(event.To.Address)).Result(); err == nil && cachedName != "" {
ensName = cachedName
gbl.Log.Infof("ensName from redis: %s", ensName)
to = toStyle.Render(ensName)
} else {
gbl.Log.Debugf("ensName not found in redis: %s", ensName)
}
// check if the ENS name is already in the cache
if name, err := cache.GetENSName(event.To.Address); err == nil && name != "" {
gbl.Log.Infof("cache | cached ENS name: %s", name)

ensName = name
}
}

if ensName == "" && viper.IsSet("api_keys.etherscan") {
if ensName := wwatcher.GetENSForAddress(nodes, event.To.Address, namesCache); ensName != "" {
if ensName := wwatcher.GetENSForAddress(ctx, nodes, event.To.Address, namesCache); ensName != "" {
gbl.Log.Debugf("ensName from etherscan: %s", ensName)
to = toStyle.Render(ensName)
event.ToENS = ensName

cache.CacheENSName(event.To.Address, ensName)
} else if event.ToENS != "" {
to = toStyle.Render(event.ToENS)
}
Expand Down
1 change: 1 addition & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func init() {
viper.SetDefault("redis.database", 0)
viper.SetDefault("redis.password", "")

viper.SetDefault("cache.names_ttl", 2*24*time.Hour)
viper.SetDefault("cache.ens_ttl", 3*24*time.Hour)
// viper.SetDefault("cache.sales_ttl", 7*24*time.Hour)
// viper.SetDefault("cache.listings_ttl", 7*24*time.Hour)
Expand Down
97 changes: 97 additions & 0 deletions internal/cache/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package cache

import (
"context"
"errors"
"time"

"github.com/benleb/gloomberg/internal/gbl"
"github.com/ethereum/go-ethereum/common"
"github.com/go-redis/redis/v8"
"github.com/spf13/viper"
)

var gbCache *GbCache

type GbCache struct {
rdb *redis.Client
addressToName map[common.Address]string
}

func New(ctx context.Context) *GbCache {
if gbCache != nil {
return gbCache
}

gCache := &GbCache{
addressToName: make(map[common.Address]string),
}

if viper.GetBool("redis.enabled") {
if client := NewRCache(ctx); client != nil {
gCache.rdb = client
}
}

return gCache
}

func (c *GbCache) GetRDB() *redis.Client {
return c.rdb
}

func (c *GbCache) CacheCollectionName(collectionAddress common.Address, collectionName string) {
c.cacheName(collectionAddress, keyContract, collectionName, viper.GetDuration("cache.names_ttl"))
}

func (c *GbCache) GetCollectionName(collectionAddress common.Address) (string, error) {
return c.getName(collectionAddress, keyContract)
}

func (c *GbCache) CacheENSName(walletAddress common.Address, ensName string) {
c.cacheName(walletAddress, keyENS, ensName, viper.GetDuration("cache.ens_ttl"))
}

func (c *GbCache) GetENSName(walletAddress common.Address) (string, error) {
return c.getName(walletAddress, keyENS)
}

func (c *GbCache) cacheName(address common.Address, keyFunc func(common.Address) string, value string, duration time.Duration) {
c.addressToName[address] = value

if c.rdb != nil {
gbl.Log.Debugf("redis | searching for: %s", keyFunc(address))

err := c.rdb.SetEX(c.rdb.Context(), keyFunc(address), value, duration).Err()

if err != nil {
gbl.Log.Warnf("redis | error while adding: %s", err.Error())
} else {
gbl.Log.Debugf("redis | added: %s -> %s", keyFunc(address), value)
}
}
}

func (c *GbCache) getName(address common.Address, keyFunc func(common.Address) string) (string, error) {
if name := c.addressToName[address]; name != "" {
return name, nil
}

if c.rdb != nil {
gbl.Log.Debugf("redis | searching for: %s", keyFunc(address))

if name, err := c.rdb.Get(c.rdb.Context(), keyFunc(address)).Result(); err == nil && name != "" {
gbl.Log.Debugf("redis | using cached name: %s", name)

c.addressToName[address] = name

return name, nil
} else {
gbl.Log.Debugf("redis | name not found in cache: %s", err)

return "", err
}
}

return "", errors.New("name not found in cache")
}
44 changes: 19 additions & 25 deletions internal/cache/redis.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
package cache

import (
"context"
"fmt"
"strings"
"sync"

"github.com/ethereum/go-ethereum/common"
"github.com/go-redis/redis/v8"
"github.com/spf13/viper"
)

var (
rdb *redis.Client
mu = &sync.RWMutex{}
)
// cache = &redisCache{
// rdb: nil,
// }
var mu = &sync.RWMutex{}

type NamespaceKey string

Expand All @@ -22,33 +24,25 @@ const (
KeyDelimiter string = ":"
)

func KeyContract(contractAddress common.Address) string {
func keyContract(contractAddress common.Address) string {
return fmt.Sprint(PrefixContractName, KeyDelimiter, contractAddress.Hex())
}

func KeyENS(address common.Address) string {
func keyENS(address common.Address) string {
return fmt.Sprint(PrefixENS, KeyDelimiter, address.Hex())
}

func GetRedisClient() *redis.Client {
mu.Lock()
if rdb == nil {
redisHost := viper.GetString("redis.host")
redisPort := viper.GetInt("redis.port")
redisPassword := viper.GetString("redis.password")
redisDatabase := viper.GetInt("redis.database")

rdb = redis.NewClient(&redis.Options{
Addr: fmt.Sprint(redisHost, ":", redisPort),
Password: redisPassword,
DB: redisDatabase,
})

if viper.GetBool("log.debug") {
fmt.Println(rdb.Info(rdb.Context()).Val())
}
}
mu.Unlock()
func NewRCache(ctx context.Context) *redis.Client {
rdb := redis.NewClient(&redis.Options{
Addr: strings.Join([]string{
viper.GetString("redis.host"),
fmt.Sprint(viper.GetInt("redis.port")),
}, ":"),
Password: viper.GetString("redis.password"),
DB: viper.GetInt("redis.database"),
})

rdb.WithContext(ctx)

return rdb
}
132 changes: 18 additions & 114 deletions internal/collections/collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"math/big"
"math/rand"
"sync/atomic"
"time"

"github.com/VividCortex/ewma"
"github.com/benleb/gloomberg/internal/cache"
Expand All @@ -25,6 +24,7 @@ type GbCollection struct {

ContractAddress common.Address `mapstructure:"address"`
Name string `mapstructure:"name"`
OpenseaSlug string `mapstructure:"slug"`

Show struct {
Sales bool `mapstructure:"sales"`
Expand Down Expand Up @@ -79,58 +79,43 @@ var ctx = context.Background()
// func (uc *Collection) UnmarshalBinary(data []byte) error { return json.Unmarshal(data, uc) }

func NewCollection(contractAddress common.Address, name string, nodes *gbnode.NodeCollection, source CollectionSource) *GbCollection {
// name
var collectionName string

cache := cache.New(ctx)
nameInCache := false

if name != "" {
collectionName = name
} else if viper.GetBool("redis.enabled") {
// check if the collection is already in the redis cache
if rdb := cache.GetRedisClient(); rdb != nil {
gbl.Log.Debugf("redis | searching for contract address: %s", contractAddress.String())

if redisName, err := rdb.Get(ctx, cache.KeyContract(contractAddress)).Result(); err == nil && redisName != "" {
gbl.Log.Debugf("redis | using cached contractName: %s", redisName)
// check if the collection is already in the cache
if name, err := cache.GetCollectionName(contractAddress); err == nil && name != "" {
gbl.Log.Infof("cache | cached collection name: %s", name)

collectionName = redisName
}
collectionName = name
nameInCache = true
}
} else {
collectionName = nodes.GetRandomNode().GetCollectionName(contractAddress)
}

// redis
if viper.GetBool("redis.enabled") {
if rdb := cache.GetRedisClient(); rdb != nil {
err := rdb.SetEX(ctx, cache.KeyContract(contractAddress), collectionName, time.Hour*48).Err()
if collectionName != "" && !nameInCache {
// cache collection name
if viper.GetBool("redis.enabled") {
gbl.Log.Infof("cache | caching collection name: %s", collectionName)

if err != nil {
gbl.Log.Infof("redis | error while adding: %s", err.Error())
} else {
gbl.Log.Debugf("redis | added: %s -> %s", contractAddress.Hex(), collectionName)
}
cache.CacheCollectionName(contractAddress, collectionName)
}
}

// if uc.source == Wallet || uc.source == Stream {
// uc.Show.Sales = viper.GetBool("show.sales")
// uc.Show.Mints = viper.GetBool("show.mints")
// uc.Show.Transfers = viper.GetBool("show.transfers")

// if viper.IsSet("api_keys.opensea") {
// uc.Show.Listings = viper.GetBool("show.listings")
// }
// }

collection := GbCollection{
ContractAddress: contractAddress,
Name: collectionName,
// Metadata: collectionMetadata,
Metadata: &gbnode.CollectionMetadata{},
Source: source,

Metadata: &gbnode.CollectionMetadata{},
Source: source,

ArtificialFloor: ewma.NewMovingAverage(),
SaLiRa: ewma.NewMovingAverage(),
// Show: show,
}

go func() {
Expand All @@ -156,10 +141,6 @@ func NewCollection(contractAddress common.Address, name string, nodes *gbnode.No
// generate the collection color based on the contract address if none given
collection.generateColorsFromAddress()

// // initialize the moving averages
// uc.artificialFloor = ewma.NewMovingAverage()
// uc.saLiRa = ewma.NewMovingAverage()

// initialize the counters
collection.ResetStats()

Expand Down Expand Up @@ -248,80 +229,3 @@ func (uc *GbCollection) generateColorsFromAddress() {
uc.Colors.Secondary = secondary
}
}

// func GetCollectionMetadata(contractAddress common.Address) *models.CollectionMetadata {
// // get the contractERC721 ABIs
// contractERC721, err := abis.NewERC721v3(contractAddress, p.Client)
// if err != nil {
// gbl.Log.Error(err)
// }

// // collection name
// collectionName := ""

// // check if the collection is already in the redis cache
// if viper.GetBool("redis.enabled") {
// if rdb := cache.GetRedisClient(); rdb != nil {
// gbl.Log.Debugf("redis | searching for contract address: %s", contractAddress.String())

// if name, err := rdb.Get(ctx, cache.KeyContract(contractAddress)).Result(); err == nil && name != "" {
// gbl.Log.Debugf("redis | using cached contractName: %s", name)

// collectionName = name
// }
// }
// }

// // if not found in redis, we call the contract method to get the name
// if collectionName == "" {
// if name, err := contractERC721.Name(&bind.CallOpts{}); err == nil {
// collectionName = name

// if viper.GetBool("redis.enabled") {
// if rdb := cache.GetRedisClient(); rdb != nil {
// err := rdb.SetEX(ctx, cache.KeyContract(contractAddress), collectionName, time.Hour*48).Err()

// if err != nil {
// gbl.Log.Infof("redis | error while adding: %s", err.Error())
// } else {
// gbl.Log.Debugf("redis | added: %s -> %s", contractAddress.Hex(), collectionName)
// }
// }
// }
// } else {
// collectionName = style.ShortenAddress(&contractAddress)
// }
// }

// // collection total supply
// collectionTotalSupply := uint64(0)
// if totalSupply, err := contractERC721.TotalSupply(&bind.CallOpts{}); err == nil {
// collectionTotalSupply = totalSupply.Uint64()
// }

// // collection symbol
// collectionSymbol := ""
// if symbol, err := contractERC721.Symbol(&bind.CallOpts{}); err == nil {
// collectionSymbol = symbol
// }

// // collection token uri
// collectionTokenURI := ""
// if tokenURI, err := contractERC721.TokenURI(&bind.CallOpts{}, big.NewInt(1)); err == nil {
// collectionTokenURI = tokenURI
// }

// return &models.CollectionMetadata{
// Name: collectionName,
// Symbol: collectionSymbol,
// TotalSupply: collectionTotalSupply,
// TokenURI: collectionTokenURI,
// }
// }

// type CollectionMetadata struct {
// OpenseaSlug string `mapstructure:"slug"`
// Symbol string `mapstructure:"symbol"`
// TotalSupply uint64 `mapstructure:"total_supply"`
// TokenURI string `mapstructure:"token_uri"`
// }
Loading

0 comments on commit 04ad063

Please sign in to comment.