-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
130 lines (108 loc) · 2.86 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/chuxorg/chux-parser/logging"
"github.com/chuxorg/chux-parser/parsing"
"github.com/chuxorg/chux-parser/s3"
)
var logFileMutex sync.Mutex
var logFile *os.File
var logger *logging.Logger
func main() {
os.Setenv("AWS_REGION", "us-east-1")
os.Setenv("LOG_LEVEL", "0")
logger = logging.NewLogger(logging.LogLevelDebug)
err := fetchAndSetSecrets("dev/secrets")
if err != nil {
log.Fatalf("failed to fetch and set secrets: %v", err)
}
fmt.Print("Setting up logging...")
setUpLogging()
defer closeLogFile()
logger.Debug("Logging set up")
logger.Info("Logging set up")
bucket := s3.New(
s3.WithLogger(logger),
)
files, err := bucket.Download()
if err != nil {
logger.Error("Failed to download files from S3", err)
panic(err)
}
parser := parsing.New(
parsing.WithLogger(logger),
)
logger.Info("Parsing %d Products and Articles", len(files))
startTime := time.Now()
for _, f := range files {
parser.Parse(f)
}
elapsedTime := time.Since(startTime).Seconds()
logger.Info("Parsed %d Articles and Products in %d seconds", len(files), elapsedTime)
}
func setUpLogging() {
var err error
logDir := "logs/chux-cprs/"
err = os.MkdirAll(logDir, 0755) // Set permissions to 0755
if err != nil {
log.Fatalf("Error creating log directory: %v", err)
}
// Open the log file
logFilePath := filepath.Join(logDir, "chux-parser.log")
logFile, err = os.OpenFile(logFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
log.Fatalf("Error opening log file: %v", err)
}
logLevel, err := strconv.Atoi(os.Getenv("LOG_LEVEL"))
if err != nil {
logLevel = int(logging.LogLevelInfo)
}
logger = logging.NewLogger(logging.LogLevel(logLevel))
logger.SetOutput(logFile)
}
func fetchAndSetSecrets(secretID string) error {
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
})
if err != nil {
return fmt.Errorf("failed to create AWS session: %v", err)
}
svc := secretsmanager.New(sess)
input := &secretsmanager.GetSecretValueInput{
SecretId: aws.String(secretID),
}
result, err := svc.GetSecretValue(input)
if err != nil {
return fmt.Errorf("failed to get secret value: %v", err)
}
var secrets map[string]string
err = json.Unmarshal([]byte(*result.SecretString), &secrets)
if err != nil {
return fmt.Errorf("failed to unmarshal secrets: %v", err)
}
for key, value := range secrets {
err = os.Setenv(key, value)
fmt.Printf("Setting env variable: %s\n", key)
if err != nil {
return fmt.Errorf("failed to set environment variable: %v", err)
}
}
return nil
}
func closeLogFile() {
logFileMutex.Lock()
defer logFileMutex.Unlock()
if logFile != nil {
logFile.Close()
}
}