-
Notifications
You must be signed in to change notification settings - Fork 6
/
content.go
187 lines (140 loc) · 5.17 KB
/
content.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
package stitcher
import (
"context"
"html/template"
"io/ioutil"
"log"
"net/url"
"strings"
"time"
"github.com/Masterminds/sprig"
"github.com/PuerkitoBio/goquery"
"github.com/bradfitz/iter"
"github.com/mailgun/groupcache/v2"
"github.com/valyala/fasttemplate"
)
// TODO The endpoint code needs to be re-done... getting crufty.
type requestContextKey string
// Content is a piece of content representing a page or portion of a page
type Content struct {
Source string `hcl:"source,optional"` // URL to fetch the main source
Selector string `hcl:"select,optional"` // CSS Selector to extract content from - optional
Replacements []Replacement `hcl:"replacement,block"` // May be empty
CacheKey string `hcl:"cache,optional"`
CacheTTL string `hcl:"ttl,optional"`
Template string `hcl:"template,optional"` // Go template source -- URL
JSON string `hcl:"json,optional"` // Used by templates to retrieve data -- URL
parsedTemplate *template.Template // Cached/preparsed template... parsed on first use.
//Options hcl.Body `hcl:",remain"`
//FetchData map[string]string `hcl:"rules"`
}
// Fetch returns the rendered content (or from endpoint if endpoint is configured for the end point)
func (c *Content) Fetch(site *Host, contextdata map[string]interface{}) (string, error) {
//log.Printf("Content From: %+v\n", endpoint)
if c.Caching() {
var content string
var contextvalue = ContentContextValue{Site: site, Content: c, ContextData: contextdata}
ctx, cancel := context.WithTimeout(context.WithValue(context.Background(), requestContextKey("request"), contextvalue),
time.Millisecond*500)
defer cancel()
if err := site.Cache.Get(ctx, c.InterpolatedCacheKey(contextdata), groupcache.StringSink(&content)); err != nil {
log.Printf("Error getting from cache: %v\n", err)
return "", err
}
return content, nil
}
return c.Render(site, contextdata)
}
// Render loads content from SourceURI and merges an fragements
// into the resulting document and returns the string representation
func (c *Content) Render(site *Host, contextdata map[string]interface{}) (string, error) {
var renderedContent string
content, err2 := c.fetcher(contextdata).Fetch()
if err2 != nil {
log.Println(err2)
return "", err2
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
if err != nil {
log.Println(err)
return "", err
}
// Might be empty but otherwise, get and merge the content for each one
// TODO Add context handling and cancelation
for _, merge := range c.Replacements {
// Retrieve fragement.Source content
fragment, _ := merge.Content.Fetch(site, contextdata)
// Find the insertion point in sourceFile at merge.InsertAt
insertSelection := doc.Find(merge.At) // Potentially costly, look into caching the source and insert selection points!?!
// Insert the extracted content
insertSelection.ReplaceWithHtml(fragment)
}
// By having the selector we can treat endpoints as a component
if c.Selector != "" {
// Get content at Selector
renderedContent, err = doc.Find(c.Selector).Html()
} else {
renderedContent, err = doc.Html()
}
return renderedContent, err
}
// Factory method to return a fetcher for the end point
func (c *Content) fetcher(contextdata map[string]interface{}) DocumentFetcher {
var fetcher DocumentFetcher
fetcher = &StringFetcher{Body: ""} // Default to empty string
if c.Source != "" {
t := fasttemplate.New(c.Source, "{{", "}}")
s := t.ExecuteString(contextdata)
u, err := url.Parse(s)
if err != nil {
// TODO Log the error
return &StringFetcher{Body: ""}
}
switch u.Scheme {
case "": // Pathname eg: "path/to/file" or "/abs/path/to/file"
fetcher = &FileFetcher{Path: u.Path}
case "string": // Inline string data eg "string:This is my String"
fetcher = &StringFetcher{Body: u.Opaque}
default: // Any other supported uri/url (ie http/https)
fetcher = &URIFetcher{URI: s}
}
}
if c.Template != "" {
if true /*endpoint.parsedTemplate == nil*/ {
templateBytes, _ := ioutil.ReadFile(c.Template)
templateContents := string(templateBytes)
funcs := sprig.GenericFuncMap()
funcs["N"] = iter.N
funcs["unescape"] = unescape
c.parsedTemplate = template.Must(template.New(c.Template).Funcs(template.FuncMap(funcs)).Parse(templateContents))
}
// interpolate the path for the any JSON source
t := fasttemplate.New(c.JSON, "{{", "}}")
json := t.ExecuteString(contextdata)
return &RenderedTemplateFetcher{Template: c.parsedTemplate,
DataURL: json,
SourceFetcher: fetcher,
RequestContext: contextdata,
}
}
// Fallback to empty string
return fetcher
}
func unescape(s string) template.HTML {
return template.HTML(s)
}
// ContentContextValue is passed via Context.WithValue() to the endpoint Getter Func
type ContentContextValue struct {
Site *Host
Content *Content
ContextData map[string]interface{}
}
// Caching returns true if we are to use the endpoint
func (c *Content) Caching() bool {
return c.CacheKey != ""
}
// InterpolatedCacheKey returns the interpolated endpoint key
func (c *Content) InterpolatedCacheKey(contextData map[string]interface{}) string {
t := fasttemplate.New(c.CacheKey, "{{", "}}")
return t.ExecuteString(contextData)
}