Skip to content

Commit

Permalink
[#] lambda_worker: prefetching twitter OAuth key job
Browse files Browse the repository at this point in the history
  • Loading branch information
nykma committed Sep 17, 2023
1 parent 3b0a266 commit 3426ffb
Show file tree
Hide file tree
Showing 9 changed files with 580 additions and 428 deletions.
26 changes: 7 additions & 19 deletions cmd/lambda/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"os"

"github.com/akrylysov/algnhsa"
"github.com/aws/aws-sdk-go-v2/aws"
Expand All @@ -13,7 +12,9 @@ import (
myconfig "github.com/nextdotid/proof_server/config"
"github.com/nextdotid/proof_server/controller"
"github.com/nextdotid/proof_server/model"
"github.com/nextdotid/proof_server/util"
"github.com/nextdotid/proof_server/util/sqs"
utilS3 "github.com/nextdotid/proof_server/util/s3"
"github.com/nextdotid/proof_server/validator/activitypub"
"github.com/nextdotid/proof_server/validator/das"
"github.com/nextdotid/proof_server/validator/discord"
Expand All @@ -33,7 +34,8 @@ var (
)

func init_db(cfg aws.Config) {
model.Init(false) // TODO: should read auto migrate flag from ENV
shouldMigrate := util.GetE("DB_MIGRATE", "false")
model.Init(shouldMigrate == "true")
}

func init_sqs(cfg aws.Config) {
Expand Down Expand Up @@ -70,6 +72,7 @@ func init() {
init_db(cfg)
init_sqs(cfg)
init_validators()
utilS3.Init(cfg)
controller.Init()
}

Expand All @@ -81,8 +84,8 @@ func init_config_from_aws_secret() {
if initialized {
return
}
secret_name := getE("SECRET_NAME", "")
region := getE("SECRET_REGION", "")
secret_name := util.GetE("SECRET_NAME", "")
region := util.GetE("SECRET_REGION", "")

// Create a Secrets Manager client
cfg, err := config.LoadDefaultConfig(
Expand Down Expand Up @@ -116,18 +119,3 @@ func init_config_from_aws_secret() {
}
initialized = true
}

func getE(env_key, default_value string) string {
result := os.Getenv(env_key)
if len(result) == 0 {
if len(default_value) > 0 {
return default_value
} else {
logrus.Fatalf("ENV %s must be given! Abort.", env_key)
return ""
}

} else {
return result
}
}
69 changes: 37 additions & 32 deletions cmd/lambda_worker/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"

"github.com/aws/aws-lambda-go/events"
Expand All @@ -20,6 +19,8 @@ import (
myconfig "github.com/nextdotid/proof_server/config"
"github.com/nextdotid/proof_server/model"
"github.com/nextdotid/proof_server/types"
"github.com/nextdotid/proof_server/util"
utilS3 "github.com/nextdotid/proof_server/util/s3"
"github.com/nextdotid/proof_server/validator/activitypub"
"github.com/nextdotid/proof_server/validator/das"
"github.com/nextdotid/proof_server/validator/discord"
Expand All @@ -38,6 +39,7 @@ import (
)

var (
awsConfig aws.Config
initialized = false
wallet *goar.Wallet
)
Expand Down Expand Up @@ -81,7 +83,7 @@ func handler(ctx context.Context, sqs_event events.SQSEvent) (events.SQSEventRes
}
case types.QueueActions.TwitterOAuthTokenAcquire:
{
err := twitterRetrieveOAuthToken()
err := twitterRefreshOAuthToken()
if err != nil {
// Ignore errors for now
fmt.Printf("Error when retrieving Twitter OAuth key: %s", err.Error())
Expand Down Expand Up @@ -134,7 +136,7 @@ func arweave_upload_many(personas []string) error {

for _, pc := range chains {
if pc.ArweaveID != "" {
continue

}

previous, ok := lo.Find(chains, func(item *model.ProofChain) bool {
Expand Down Expand Up @@ -241,8 +243,9 @@ func revalidateSingle(ctx context.Context, message *types.QueueMessage) error {
return proof.Revalidate()
}

func initDB(cfg aws.Config) {
model.Init(false) // TODO: should read auto migrate from ENV
func initDB() {
shouldMigrate := util.GetE("DB_MIGRATE", "false")
model.Init(shouldMigrate == "true")
}

// func init_sqs(cfg aws.Config) {
Expand All @@ -264,7 +267,8 @@ func initValidators() {
}

func init() {
cfg, err := config.LoadDefaultConfig(
var err error
awsConfig, err = config.LoadDefaultConfig(
context.Background(),
config.WithRegion("ap-east-1"),
)
Expand All @@ -275,7 +279,7 @@ func init() {
initConfigFromAWSSecret()
logrus.SetLevel(logrus.InfoLevel)

initDB(cfg)
initDB()
// init_sqs(cfg)
initValidators()
}
Expand All @@ -284,8 +288,8 @@ func initConfigFromAWSSecret() {
if initialized {
return
}
secretName := getE("SECRET_NAME", "")
region := getE("SECRET_REGION", "")
secretName := util.GetE("SECRET_NAME", "")
region := util.GetE("SECRET_REGION", "")

// Create a Secrets Manager client
cfg, err := config.LoadDefaultConfig(
Expand Down Expand Up @@ -327,33 +331,34 @@ func initConfigFromAWSSecret() {
initialized = true
}

func getE(envKey, defaultValue string) string {
result := os.Getenv(envKey)
if len(result) == 0 {
if len(defaultValue) > 0 {
return defaultValue
} else {
logrus.Fatalf("ENV %s must be given! Abort.", envKey)
return ""
}

} else {
return result
func twitterRefreshOAuthToken() (err error) {
const VALID_TOKEN_AMOUNT = 5
if err = utilS3.Init(awsConfig); err != nil {
logrus.Fatalf("Error during initializing S3 client: %s", err.Error())
}
}

func twitterRetrieveOAuthToken() (err error) {
type TokenList struct {
Tokens []twitter.Tokens `json:"tokens"`
ctx := context.Background()
tokens, err := twitter.GetTokenListFromS3(ctx)
if err != nil {
logrus.Fatalf("Error when loading Twitter token list from S3: %s", err.Error())
}
// TODO: Retrieve existed token from a storage space (i.e., KV / S3)

tokens, err := twitter.GenerateOauthToken()
if err != nil {
return err
validTokens := lo.Filter(tokens.Tokens, func(token twitter.Token, _index int) bool {
return !token.IsExpired()
})
if len(validTokens) < VALID_TOKEN_AMOUNT {
// Generate a new one
newToken, err := twitter.GenerateOauthToken()
if err != nil {
return err
}
fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens)
validTokens = append(validTokens, *newToken)
newTokenList := twitter.TokenList{
Tokens: validTokens,
}
newTokenListJSON, _ := newTokenList.ToJSON()
utilS3.PutToS3(ctx, twitter.TWITTER_TOKEN_LIST_FILENAME, newTokenListJSON)
}
fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens)

// TODO: save new token to a storage space (i.e., KV / S3)
return nil
}
16 changes: 11 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.21

require (
github.com/aws/aws-lambda-go v1.31.1
github.com/aws/aws-sdk-go-v2 v1.16.5
github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/config v1.15.4
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6
github.com/everFinance/goar v1.4.2
Expand All @@ -26,15 +26,20 @@ require (
contrib.go.opencensus.io/exporter/stackdriver v0.13.4 // indirect
filippo.io/edwards25519 v1.0.0-rc.1 // indirect
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 // indirect
github.com/aws/smithy-go v1.11.3 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect
github.com/blendle/zapdriver v1.3.1 // indirect
github.com/btcsuite/btcd v0.22.0-beta // indirect
Expand Down Expand Up @@ -148,6 +153,7 @@ require (

require (
github.com/akrylysov/algnhsa v0.12.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6
github.com/bwmarrin/discordgo v0.25.0
github.com/ethereum/go-ethereum v1.10.25
Expand Down
27 changes: 22 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -75,24 +75,40 @@ github.com/aws/aws-lambda-go v1.31.1/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+K
github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI=
github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48=
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
github.com/aws/aws-sdk-go-v2/config v1.15.4 h1:P4mesY1hYUxru4f9SU0XxNKXmzfxsD0FtMIPRBjkH7Q=
github.com/aws/aws-sdk-go-v2/config v1.15.4/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4=
github.com/aws/aws-sdk-go-v2/credentials v1.12.0 h1:4R/NqlcRFSkR0wxOhgHi+agGpbEr5qMCjn7VqUIJY+E=
github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/CljkTrNVWBSK9pkqOPUuijid4A=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 h1:FP8gquGeGHHdfY6G5llaMQDF+HAf20VKc8opRwmjf04=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 h1:6cZRymlLEIlDTEB0+5+An6Zj1CKt6rSE69tOmFeu1nk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 h1:m+mxqLIrGq7GJo5qw4rHn8BbUqHrvxvwFx54N1Pglvw=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6/go.mod h1:Z+i6uqZgCOBXhNoEGoRm/ZaLsaJA9rGUAmkVKM/3+g4=
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 h1:HlEYt9p1TAQYxeB8jz3y4dmXmZevX+cJnh8OU6x0aqo=
Expand All @@ -102,8 +118,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmd
github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE=
github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8=
github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
Expand Down
47 changes: 47 additions & 0 deletions util/s3/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package s3

import (
"bytes"
"context"
"io"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/nextdotid/proof_server/util"
)

var (
s3Client *s3.Client
bucketName string
)

func Init(awsConfig aws.Config) error {
if s3Client == nil {
s3Client = s3.NewFromConfig(awsConfig)
}
bucketName = util.GetE("S3_BUCKET", "")
return nil
}

// ReadFromS3 should be called before Init()
func ReadFromS3(ctx context.Context, key string) (content []byte, err error) {
result, err := s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
if err != nil {
return []byte{}, err
}
defer result.Body.Close()
return io.ReadAll(result.Body)
}

// PutToS3 should be called before Init()
func PutToS3(ctx context.Context, key string, content []byte) (err error) {
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: bytes.NewReader(content),
})
return err
}
18 changes: 18 additions & 0 deletions util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ package util

import (
"encoding/base64"
"os"
"strconv"
"time"

"github.com/nextdotid/proof_server/util/base1024"
"github.com/sirupsen/logrus"

"golang.org/x/xerrors"
)
Expand All @@ -30,3 +32,19 @@ func DecodeString(s string) ([]byte, error) {
}
return base1024.DecodeString(s)
}

// GetE gets current system's environment variable as `string`.
// If `defaultValue` is empty, this environment key must be exist, or it will panic.
func GetE(envKey, defaultValue string) string {
result := os.Getenv(envKey)
if len(result) == 0 {
if len(defaultValue) > 0 {
return defaultValue
} else {
logrus.Fatalf("ENV %s must be given! Abort.", envKey)
return ""
}

}
return result
}

0 comments on commit 3426ffb

Please sign in to comment.