/
index.go
137 lines (118 loc) · 2.96 KB
/
index.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
package manifest
import (
"bufio"
"database/sql"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/arken/downlink/database"
)
type Result struct {
Node database.Node
Status string
}
func (m *Manifest) Index(db *database.DB) (<-chan Result, error) {
// Create results channel
results := make(chan Result, 50)
// Initialize go routine to handle indexing
go func() {
m.indexWorker(db, results)
// Close results channel
close(results)
}()
return results, nil
}
func (m *Manifest) indexWorker(db *database.DB, output chan<- Result) {
// Store current time to find files not touched during index
start := time.Now().UTC()
// Walk through entire repository directory structure to look for .ks files.
err := filepath.Walk(m.path, func(path string, info os.FileInfo, err error) error {
// On each interation of the "walk" this function will check if a manifest
// file and parse for file IDs if true.
if strings.HasSuffix(filepath.Base(path), ".ks") {
file, err := os.Open(path)
if err != nil {
return err
}
// Add filepaths all the way up to the root.
fpath := strings.TrimSuffix(strings.TrimPrefix(path, m.path), ".ks")
for {
// Check if entry is already in the database.
found, err := db.Get(fpath)
if err == nil || err != sql.ErrNoRows {
db.Update(found)
} else {
result := Result{
Status: "add",
Node: database.Node{
Path: fpath,
Name: filepath.Base(fpath),
Type: "dir",
Parent: filepath.Dir(fpath),
},
}
output <- result
}
// Break the loop at the root node
if fpath == "/" {
break
}
// Climb up the file path
fpath = filepath.Dir(fpath)
}
// Generate parent path for files in manifest file
parentPath := strings.TrimSuffix(strings.TrimPrefix(path, m.path), ".ks")
// Open the files for reading.
scanner := bufio.NewScanner(file)
// Scan one line at a time.
for scanner.Scan() {
// Split data on white space.
data := strings.Fields(scanner.Text())
// Check if entry is already in the database.
found, err := db.Get(filepath.Join(parentPath, data[1]))
if err == nil || err != sql.ErrNoRows {
db.Update(found)
continue
}
result := Result{
Status: "add",
Node: database.Node{
Path: filepath.Join(parentPath, data[1]),
Name: data[1],
Type: "file",
CID: data[0],
Parent: parentPath,
},
}
output <- result
}
// Close the file after fully parsed.
err = file.Close()
if err != nil {
return err
}
}
return nil
})
if err != nil {
log.Println(err)
}
// Check for deleted files by looking for everything not touched by the index
for i := 0; ; i++ {
nodes, err := db.GetAllOlderThan(start, 100, i)
if err != nil {
break
}
for _, file := range nodes {
output <- Result{
Status: "remove",
Node: file,
}
}
}
if err != nil && err != sql.ErrNoRows {
log.Println(err)
}
}