-
-
Notifications
You must be signed in to change notification settings - Fork 26
/
reindexusers.go
101 lines (85 loc) · 3.27 KB
/
reindexusers.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
package cmd
import (
"math"
"strconv"
"sync"
modelsutils "atomys.codes/stud42/internal/models"
modelgen "atomys.codes/stud42/internal/models/generated"
"atomys.codes/stud42/internal/pkg/searchengine"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)
// reindexusersCmd represents the reindexusers command
var reindexusersCmd = &cobra.Command{
Use: "reindexusers",
Short: "Do a re-indexation of the users on the meilisearch index",
Long: `This operation will re-index all the users on the meilisearch index.
This operation is useful when the meilisearch index is corrupted or when the
meilisearch index is not up to date. This operation will take a long time to
complete. This operation will drop the meilisearch index and recreate it with
all the users.`,
PreRun: func(cmd *cobra.Command, args []string) {
if err := modelsutils.Connect(); err != nil {
log.Fatal().Err(err).Msg("failed to connect to database")
}
if err := modelsutils.Migrate(); err != nil {
log.Fatal().Err(err).Msg("failed to migrate database")
}
searchengine.Initizialize()
},
Run: func(cmd *cobra.Command, args []string) {
log.Info().Msg("Prepare the re-indexation of the users")
meiliClient := searchengine.NewClient()
// Delete the index
if _, err := meiliClient.DeleteIndex(searchengine.IndexUser); err != nil {
log.Fatal().Err(err).Msg("failed to delete the index")
}
if err := meiliClient.EnsureAllIndexes(); err != nil {
log.Fatal().Err(err).Msg("failed to ensure all indexes")
}
// Re-index all the users
batchSize, err := strconv.Atoi(cmd.Flag("batch_size").Value.String())
if err != nil || batchSize < 1 {
log.Fatal().Err(err).Msg("Cannot cast batch_size flag. batch_size msut be a positive integer")
}
usersCount := modelsutils.Client().User.Query().CountX(cmd.Context())
batchCount := int(math.Ceil(float64(usersCount) / float64(batchSize)))
log.Info().
Int("usersCount", usersCount).
Int("batchSize", batchSize).
Int("batchCount", batchCount).
Msg("Start the re-indexation of the users")
for i := 0; i*batchSize < usersCount; i += 1 {
users := modelsutils.Client().User.Query().Offset(i * batchSize).Limit(batchSize).AllX(cmd.Context())
log.Info().Int("batch", i+1).Int("batchCount", batchCount).Int("usersCount", len(users)).Msg("Reindexing users in batch")
var wg sync.WaitGroup
for _, user := range users {
wg.Add(1)
go func(user *modelgen.User) {
defer wg.Done()
hasOnline := user.CurrentLocationID != nil && *user.CurrentLocationID != uuid.Nil
document := &searchengine.UserDocument{
ID: user.ID,
CurrentCampusID: user.CurrentCampusID,
DuoLogin: user.DuoLogin,
FirstName: user.FirstName,
UsualFirstName: user.UsualFirstName,
LastName: user.LastName,
HasOnline: &hasOnline,
}
if err := meiliClient.UpdateUserDocument(cmd.Context(), document); err != nil {
log.Fatal().Err(err).Msg("failed to index user")
}
}(user)
}
// Wait for the goroutine to finish
wg.Wait()
}
log.Info().Msg("End of the re-indexation of the users")
},
}
func init() {
operationsCmd.AddCommand(reindexusersCmd)
operationsCmd.PersistentFlags().IntP("batch_size", "b", 50, "Batch size of the reindex")
}