-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
native.go
138 lines (111 loc) · 3.41 KB
/
native.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
package migrator
import (
"bytes"
"encoding/json"
"io"
"log"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
const natvieVersion = 1
// Native implements exporter and importer for internal store format
// {"version": 1, comments:[{...}\n,{}], meta: {meta}}
// each comments starts from the new line
type Native struct {
DataStore Store
}
type meta struct {
Version int `json:"version"`
Users []service.UserMetaData `json:"users"`
Posts []service.PostMetaData `json:"posts"`
}
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
// The final file is a valid json
func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
if err = n.exportMeta(siteID, w); err != nil {
return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID)
}
topics, err := n.DataStore.List(siteID, 0, 0)
if err != nil {
return 0, err
}
log.Printf("[DEBUG] exporting %d topics", len(topics))
commentsCount := 0
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
topic := topics[i]
comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
if err != nil {
return commentsCount, e
}
for _, comment := range comments {
buf := &bytes.Buffer{}
enc := json.NewEncoder(buf)
enc.SetEscapeHTML(false)
if err = enc.Encode(comment); err != nil {
return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
}
if _, err = w.Write(buf.Bytes()); err != nil {
return commentsCount, errors.Wrap(err, "can't write comment data")
}
commentsCount++
}
}
log.Printf("[DEBUG] exported %d comments", commentsCount)
return commentsCount, nil
}
// exportMeta appends user and post metas to exported stream
func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
m := meta{Version: natvieVersion}
m.Users, m.Posts, err = n.DataStore.Metas(siteID)
if err != nil {
return errors.Wrap(err, "can't get meta")
}
if err := json.NewEncoder(w).Encode(m); err != nil {
return errors.Wrap(err, "can't encode meta")
}
return nil
}
// Import comments from json strings produced by Remark.Export
func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
m := meta{}
dec := json.NewDecoder(reader)
if err = dec.Decode(&m); err != nil {
return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
}
if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
return 0, errors.Errorf("unexpected import file version %d", m.Version)
}
if err = n.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
failed := 0
total, comments := 0, 0
for {
comment := store.Comment{}
err = dec.Decode(&comment)
if err == io.EOF {
break
}
total++
if err != nil {
failed++
continue
}
if _, err = n.DataStore.Create(comment); err != nil {
failed++
log.Printf("[WARN] can't write %+v to store, %s", comment, err)
continue
}
comments++
if comments%1000 == 0 {
log.Printf("[DEBUG] imported %d comments", comments)
}
}
if failed > 0 {
return comments, errors.Errorf("failed to save %d comments", failed)
}
log.Printf("[INFO] imported %d comments from %d records", comments, total)
err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
return comments, err
}