This repository has been archived by the owner on Oct 5, 2023. It is now read-only.
-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
sync.go
119 lines (99 loc) · 3.31 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
package commands
import (
"fmt"
"log"
"sync"
"time"
"github.com/urfave/cli/v2"
"github.com/wilhelm-murdoch/go-collection"
"github.com/wilhelm-murdoch/go-stash/client"
"github.com/wilhelm-murdoch/go-stash/config"
"github.com/wilhelm-murdoch/go-stash/ingest"
"github.com/wilhelm-murdoch/go-stash/models"
"github.com/wilhelm-murdoch/go-stash/queries"
"github.com/wilhelm-murdoch/go-stash/writers"
)
// SyncHandler is responsible for fetching content from the specified
// Hashnode website and author:
func SyncHandler(c *cli.Context, cfg *config.Configuration) error {
client := client.New()
var since time.Time
since = time.Now()
if c.String("since") != "" {
rewind, err := time.ParseDuration(c.String("since"))
if err != nil {
return fmt.Errorf("expected --since value to use format of 10s, 10m or 10h, but got `%s` instead", c.String("since"))
}
log.Printf("fetching content from `%s` ago due to usage of --since\n", c.String("since"))
since = since.Add(-rewind)
} else {
rewind, _ := time.ParseDuration("99999h")
log.Println("--since flag not used, so fetching content from the beginning")
since = since.Add(-rewind)
}
currentPage := 0
wg := new(sync.WaitGroup)
for {
result, err := client.Execute(queries.New("GetTimeline", queries.GetTimeline, queries.TimelineUnmarshaler, c.String("username"), currentPage))
if err != nil {
return err
}
if len(result.([]models.Post)) == 0 {
log.Println("done paging")
break
}
log.Printf("searching page: %d\n", currentPage+1)
// Search publication for any posts that have been added, or updated,
// between now and `since`. All results are sent to the post ingestion
// handler:
for _, post := range result.([]models.Post) {
dateAdded, _ := time.Parse(time.RFC3339, post.DateAdded)
dateUpdated, _ := time.Parse(time.RFC3339, post.DateUpdated)
if dateAdded.After(since) || dateUpdated.After(since) {
wg.Add(1)
log.Printf("found post: %s", post.Slug)
go ingest.Posts.Get(post.Slug, c.String("hostname"), wg)
}
}
currentPage++
}
wg.Wait()
images := collection.New[models.Image]()
ingest.Posts.Results().Each(func(i int, p *models.Post) bool {
images.Push(p.GetImages(cfg)...)
images.PushDistinct(p.Author.GetImages(cfg)...)
return false
})
if err := writers.WriteFileCollection(images); err != nil {
return err
}
ingest.Posts.Results().Each(func(i int, p *models.Post) bool {
p.ReplaceImagePaths(cfg)
return false
})
basePathPosts := fmt.Sprintf("%s/%s", cfg.Paths.Root, cfg.Paths.Posts)
posts := ingest.Posts.Results()
if err := writers.WriteJsonCollection(basePathPosts, posts); err != nil {
return err
}
if err := writers.WriteJsonManifest(basePathPosts, posts); err != nil {
return err
}
basePathTags := fmt.Sprintf("%s/%s", cfg.Paths.Root, cfg.Paths.Tags)
tags := ingest.Posts.GroupPostsByTag()
if err := writers.WriteJsonCollection(basePathTags, tags); err != nil {
return err
}
if err := writers.WriteJsonManifest(basePathTags, tags); err != nil {
return err
}
basePathAuthors := fmt.Sprintf("%s/%s", cfg.Paths.Root, cfg.Paths.Authors)
authors := ingest.Posts.FilterDistinctAuthors()
if err := writers.WriteJsonCollection(basePathAuthors, authors); err != nil {
return err
}
if err := writers.WriteJsonManifest(basePathAuthors, authors); err != nil {
return err
}
return nil
}