/
dependency.go
166 lines (137 loc) · 3.49 KB
/
dependency.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
package graph
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"github.com/dissipative/opabinia/internal/infra/fs"
"github.com/dissipative/opabinia/internal/markdown/parser"
"golang.org/x/sync/semaphore"
)
type DependencyType int
const (
asset DependencyType = iota
markdown
)
type DependencyTree struct {
filename string
depType DependencyType
children []*DependencyTree
}
var maxConcurrentRoutines = int64(runtime.NumCPU())
// NewDependencyTree creates a new DependencyTree for the given filename.
// If the file is a local markdown file, it extracts all the links within it and
// creates child dependencies recursively.
func NewDependencyTree(filename string, paths Paths) (*DependencyTree, error) {
if !fs.IsLocalMarkdownFile(filename) {
return &DependencyTree{
filename: filename,
depType: asset,
children: nil,
}, nil
}
var children []*DependencyTree
var ee []error
data, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("%s: %w", filename, err)
}
links := parser.ExtractLinks(data)
data = nil // explicitly remove file contents from memory
errChan := make(chan error, len(links))
childChan := make(chan *DependencyTree, len(links))
wg := new(sync.WaitGroup)
sem := semaphore.NewWeighted(maxConcurrentRoutines) // limiting concurrency
for _, link := range links {
if !fs.IsLocalFile(link) {
// web urls are not needed
continue
}
path := fs.NormalizePath(filename, link)
if paths.IsSet(path) {
// circular dependency, it is ok for hypertext; just omit
continue
}
paths.Set(path)
wg.Add(1)
go func() {
defer wg.Done()
defer sem.Release(1)
err := sem.Acquire(context.Background(), 1)
if err != nil {
errChan <- err
return
}
child, err := NewDependencyTree(path, paths)
if err != nil {
errChan <- err
return
}
childChan <- child
}()
}
go func() {
wg.Wait()
close(errChan)
close(childChan)
}()
for e := range errChan {
ee = append(ee, e)
}
for ch := range childChan {
children = append(children, ch)
}
if len(ee) > 0 {
return nil, errors.Join(ee...)
}
return &DependencyTree{
filename: filename,
depType: markdown,
children: children,
}, nil
}
func (d *DependencyTree) IsMarkdown() bool {
return d.depType == markdown
}
// processDependency processes the DependencyTree, rendering it to html if it is a markdown file
// and copying it to a destination directory.
// If it's a markdown file, it also processes its child dependencies recursively.
// Errors encountered during processing are sent to the provided error channel.
func (d *DependencyTree) processDependency(r Renderer, dst string, errChan chan<- error, wg *sync.WaitGroup) {
defer wg.Done()
newName := filepath.Join(dst, d.filename)
// create dir for new file
err := fs.CreateDir(filepath.Dir(newName))
if err != nil && !errors.Is(err, fs.ErrDirExist) {
errChan <- err
return
}
if !d.IsMarkdown() {
err = fs.CopyFile(d.filename, newName)
if err != nil {
errChan <- err
return
}
} else {
content, err := r.RenderPage(d.filename)
if err != nil {
errChan <- err
return
}
newName = filepath.Join(dst, strings.TrimSuffix(d.filename, ".md")+".html")
err = os.WriteFile(newName, content, fs.DefaultFilePerm)
content = nil // explicitly remove content from memory as soon as possible
if err != nil {
errChan <- err
return
}
for _, child := range d.children {
wg.Add(1)
go child.processDependency(r, dst, errChan, wg)
}
}
}