This repository has been archived by the owner on Sep 16, 2021. It is now read-only.
/
concatjs.go
333 lines (299 loc) · 9.45 KB
/
concatjs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
// Package concatjs provides a simple way of serving JavaScript sources in development.
package concatjs
import (
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
)
// ServeConcatenatedJS returns an http.Handler that serves the JavaScript files
// listed in manifestPath in one concatenated, eval separated response body.
//
// This greatly speeds up development load times due to fewer HTTP requests, but
// still for easy debugging by giving the eval'ed fragments URLs through
// sourceURL comments.
//
// Example usage:
// http.Handle("/app_combined.js",
// concatjs.ServeConcatenatedJS("my/app/web_srcs.MF", ".", [], [], nil))
//
// Relative paths in the manifest are resolved relative to the path given as root.
func ServeConcatenatedJS(manifestPath string, root string, preScripts []string, postScripts []string, fs FileSystem) http.Handler {
var lock sync.Mutex // Guards cache.
cache := NewFileCache(root, fs)
manifestPath = filepath.Join(root, manifestPath)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
files, err := manifestFiles(manifestPath)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
writeJSError(w, "Failed to read manifest: %v", err)
return
}
var writer io.Writer = w
if acceptGzip(r.Header) {
// NB: gzip is not supported in App Engine, as the header is stripped:
// https://cloud.google.com/appengine/docs/go/requests#Go_Request_headers
// CompressionLevel = 3 is a reasonable compromise between speed and compression.
gzw, err := gzip.NewWriterLevel(w, 3)
if err != nil {
log.Fatalf("Could not create gzip writer: %s", err)
}
defer gzw.Close()
writer = gzw
w.Header().Set("Content-Encoding", "gzip")
}
// Write out pre scripts
for _, s := range preScripts {
fmt.Fprint(writer, s)
// Ensure scripts are separated by a newline
fmt.Fprint(writer, "\n")
}
// Protect the cache with a lock because it's possible for multiple requests
// to be handled in parallel.
lock.Lock()
cache.WriteFiles(writer, files)
lock.Unlock()
// Write out post scripts
for _, s := range postScripts {
fmt.Fprint(writer, s)
// Ensure scripts are separated by a newline
fmt.Fprint(writer, "\n")
}
})
}
var acceptHeader = http.CanonicalHeaderKey("Accept-Encoding")
func acceptGzip(h http.Header) bool {
for _, hv := range h[acceptHeader] {
for _, enc := range strings.Split(hv, ",") {
if strings.TrimSpace(enc) == "gzip" {
return true
}
}
}
return false
}
// FileSystem is the interface to reading files from disk.
// It's abstracted into an interface to allow tests to replace it.
type FileSystem interface {
statMtime(filename string) (time.Time, error)
readFile(filename string) ([]byte, error)
}
// realFileSystem implements FileSystem by actual disk access.
type realFileSystem struct{}
func (fs *realFileSystem) statMtime(filename string) (time.Time, error) {
s, err := os.Stat(filename)
if err != nil {
return time.Time{}, err
}
return s.ModTime(), nil
}
func (fs *realFileSystem) readFile(filename string) ([]byte, error) {
return ioutil.ReadFile(filename)
}
// FileCache caches a set of files in memory and provides a single
// method, WriteFiles(), that streams them out in the concatjs format.
type FileCache struct {
fs FileSystem
root string
entries map[string]*cacheEntry
}
// NewFileCache constructs a new FileCache. Relative paths in the cache
// are resolved relative to root. fs injects file system access, and
// will use the real file system if nil.
func NewFileCache(root string, fs FileSystem) *FileCache {
if fs == nil {
fs = &realFileSystem{}
}
return &FileCache{
root: root,
fs: fs,
entries: map[string]*cacheEntry{},
}
}
type cacheEntry struct {
// err holds an error encountered while updating the entry; if
// it's non-nil, then mtime and contents are invalid.
err error
mtime time.Time
contents []byte
}
// manifestFiles parses a manifest, returning a list of the files in the manifest.
// It skips blank lines and javascript/closure/deps.js.
func manifestFiles(manifest string) ([]string, error) {
f, err := os.Open(manifest)
if err != nil {
return nil, fmt.Errorf("could not read manifest %s: %s", manifest, err)
}
defer f.Close()
return manifestFilesFromReader(f)
}
// manifestFilesFromReader is a helper for manifestFiles, split out for testing.
func manifestFilesFromReader(r io.Reader) ([]string, error) {
var lines []string
s := bufio.NewScanner(r)
for s.Scan() {
path := s.Text()
if path == "" {
continue
}
if path == "javascript/closure/deps.js" {
// Ignore/skip deps.js, it is unused due to CLOSURE_NO_DEPS = true and superseded by the
// dependency handling in this file. It's harmless, but a large download (>450 KB).
continue
}
lines = append(lines, path)
}
if err := s.Err(); err != nil {
return nil, err
}
return lines, nil
}
// writeJSError writes an error both to the log and into w as a JavaScript throw statement.
func writeJSError(w io.Writer, format string, a ...interface{}) {
log.Printf(format, a...)
fmt.Fprint(w, "throw new Error('")
fmt.Fprintf(w, format, a...)
fmt.Fprint(w, "');\n")
}
// WriteFiles updates the cache for a list of files, then streams them into an io.Writer.
func (cache *FileCache) WriteFiles(w io.Writer, files []string) error {
// Ensure the cache is up to date with respect to the on-disk state.
// Note that refreshFiles cannot fail; any errors encountering while refreshing
// are stored in the cache entry and streamed into the response.
cache.refreshFiles(files)
for _, path := range files {
if _, err := fmt.Fprintf(w, "// %s\n", path); err != nil {
return err
}
ce := cache.entries[path]
if ce.err != nil {
writeJSError(w, "loading %s failed: %s", path, ce.err)
continue
}
if _, err := w.Write(ce.contents); err != nil {
return err
}
}
return nil
}
// refresh ensures a single cacheEntry is up to date. It stat()s and
// potentially reads the contents of the file it is caching.
func (e *cacheEntry) refresh(root, path string, fs FileSystem) error {
mt, err := fs.statMtime(filepath.Join(root, path))
if err != nil {
return err
}
if e.mtime == mt && e.contents != nil {
return nil // up to date
}
contents, err := fileContents(root, path, fs)
if err != nil {
return err
}
e.mtime = mt
e.contents = contents
return nil
}
// refreshFiles stats the given files and updates the cache for them.
func (cache *FileCache) refreshFiles(files []string) {
// Stating many files asynchronously is faster on network file systems.
// Push all files that need to be stat'd into a channel and have
// a set of workers stat/read them to update the cache entry.
type workItem struct {
path string
entry *cacheEntry
}
work := make(chan workItem)
var wg sync.WaitGroup
wg.Add(len(files))
for i := 0; i < len(files); i++ {
// TODO(evanm): benchmark limiting this to fewer goroutines.
go func() {
w := <-work
w.entry.err = w.entry.refresh(cache.root, w.path, cache.fs)
wg.Done()
}()
}
for _, path := range files {
entry := cache.entries[path]
if entry == nil {
entry = &cacheEntry{}
cache.entries[path] = entry
}
work <- workItem{path, entry}
}
close(work)
wg.Wait()
}
// The maximum number of bytes of a source file to be searched for the "goog.module" declaration.
// Limited to 50,000 bytes to avoid degenerated performance on large compiled JS (e.g. a
// pre-compiled AngularJS binary).
const googModuleSearchLimit = 50 * 1000
// Matches files containing "goog.module", which have to be served slightly differently.
var googModuleRegExp = regexp.MustCompile(`(?m)^\s*goog\.module\s*\(\s*['"]`)
// fileContents returns escaped JS file contents for the given path.
// The path is resolved relative to root, but the path without root is used as the path
// in the source map.
func fileContents(root, path string, fs FileSystem) ([]byte, error) {
contents, err := fs.readFile(filepath.Join(root, path))
if err != nil {
return nil, err
}
var f bytes.Buffer
// goog.module files must be wrapped in a goog.loadModule call. Check the first X bytes of the file for it.
limit := googModuleSearchLimit
if len(contents) < limit {
limit = len(contents)
}
if googModuleRegExp.Match(contents[:limit]) {
fmt.Fprint(&f, "goog.loadModule('")
} else {
fmt.Fprint(&f, "eval('")
}
if err := writeJSEscaped(&f, contents); err != nil {
log.Printf("Failed to write file contents of %s: %s", path, err)
return nil, err
}
fmt.Fprintf(&f, "\\n\\n//# sourceURL=http://concatjs/%s\\n');\n", path)
return f.Bytes(), nil
}
// writeJSEscaped writes contents into the given writer, escaping for content in
// a single quoted JavaScript string.
func writeJSEscaped(out io.Writer, contents []byte) error {
// template.JSEscape escapes whitespace and line breaks to bulky six-character
// escapes, substantially blowing up response size, and is also a bit slower.
// As this also doesn't need safe escaping, this code just rather escapes itself.
for _, b := range contents {
switch b {
case '\n':
if _, err := out.Write([]byte("\\n")); err != nil {
return err
}
case '\r':
if _, err := out.Write([]byte("\\r")); err != nil {
return err
}
case '\\', '\'':
if _, err := out.Write([]byte{'\\'}); err != nil {
return err
}
fallthrough
default:
if _, err := out.Write([]byte{b}); err != nil {
return err
}
}
}
return nil
}