/
browser.clj
728 lines (598 loc) · 24.3 KB
/
browser.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
(ns shadow.build.targets.browser
(:refer-clojure :exclude (flush))
(:require [clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.data.json :as json]
[clojure.string :as str]
[clojure.set :as set]
[clojure.java.shell :as sh]
[clojure.edn :as edn]
[shadow.cljs.repl :as repl]
[shadow.cljs.util :as util]
[shadow.build.api :as build-api]
[shadow.build :as build]
[shadow.build.targets.shared :as shared]
[shadow.build.config :as config]
[shadow.build.output :as output]
[shadow.build.closure :as closure]
[shadow.build.data :as data]
[shadow.build.log :as log]
[shadow.core-ext :as core-ext]
[cljs.compiler :as cljs-comp]
[shadow.build.npm :as npm]
[shadow.cljs.devtools.server.npm-deps :as npm-deps]
[shadow.build.log :as build-log]))
(s/def ::module-loader boolean?)
(s/def ::target
(s/keys
:req-un
[::shared/modules]
:opt-un
[::module-loader
::shared/output-dir
::shared/asset-path
::shared/public-dir
::shared/public-path
::shared/devtools]))
(defmethod config/target-spec :browser [_]
(s/spec ::target))
(defmethod config/target-spec `process [_]
(s/spec ::target))
(def default-browser-config
{:output-dir "public/js"
:asset-path "/js"})
(defn json [obj]
(json/write-str obj :escape-slash false))
(defn module-loader-data [{::build/keys [mode] :keys [build-options] :as state}]
(let [release?
(= :release mode)
{:keys [asset-path cljs-runtime-path]}
build-options
build-modules
(or (::closure/modules state)
(:build-modules state))
[loader-module & modules]
build-modules
modules
(remove :web-worker modules)
loader-sources
(into #{} (:sources loader-module))
module-uris
(reduce
(fn [m {:keys [module-id foreign-files sources] :as module}]
(let [uris
(if release?
[(str asset-path "/" (:output-name module))]
;; :dev, never bundles foreign
(->> sources
(remove loader-sources)
(map (fn [src-id]
(let [{:keys [output-name] :as rc}
(data/get-source-by-id state src-id)]
(str asset-path "/" cljs-runtime-path "/" output-name))))
(into [])))]
(assoc m module-id uris)))
{}
modules)
module-infos
(reduce
(fn [m {:keys [module-id depends-on]}]
(assoc m module-id (disj depends-on (:module-id loader-module))))
{}
modules)]
{:module-uris module-uris
:module-infos module-infos}))
(defn inject-repl-client
[{:keys [entries] :as module-config} state build-config]
(let [{:keys [enabled]}
(:devtools build-config)
entries
(-> []
(cond->
(not (false? enabled))
(into '[cljs.user shadow.cljs.devtools.client.browser]))
(into entries))]
(assoc module-config :entries entries)))
(defn inject-devtools-console [{:keys [entries] :as module-config} state build-config]
(cond
(or (false? (get-in build-config [:devtools :console-support]))
(->> (get-in build-config [:devtools :preloads])
;; automatically back off if cljs-devtools is used manually
(filter #(str/starts-with? (str %) "devtools."))
(seq)))
module-config
;; automatically inject `cljs-devtools` when found on the classpath
(io/resource "devtools/preload.cljs")
(assoc module-config :entries (into '[devtools.preload] entries))
:else
(assoc module-config :entries (into '[shadow.cljs.devtools.client.console] entries))
))
(defn inject-preloads [{:keys [entries] :as module-config} state build-config]
(let [preloads (get-in build-config [:devtools :preloads])]
(if-not (seq preloads)
module-config
(assoc module-config :entries (into (vec preloads) entries)))))
(defn pick-default-module-from-config [modules]
(or (reduce-kv
(fn [default module-id {:keys [depends-on] :as module-config}]
(cond
(:default module-config)
(reduced module-id)
(seq depends-on)
default
;; empty depends-on, but encountered one previously
;; FIXME: be smarter about detecting default modules, can look at entries for cljs.core or shadow.loader
default
(throw (ex-info "two modules without deps, please specify which one is the default" {:a default
:b module-id}))
:else
module-id
))
nil
modules)
(throw (ex-info "all modules have deps, can't find default" {}))
))
(defn wrap-output [{:keys [default prepend append] :as module-config} state]
(let [ppns (get-in state [:compiler-options :rename-prefix-namespace])]
(-> module-config
(assoc :prepend (str prepend
(when (and default (seq ppns))
(str "var " ppns " = {};\n"))
"(function(){\n")
:append (str append "\n}).call(this);")))))
(defn apply-output-wrapper
([state]
(update state ::closure/modules apply-output-wrapper state))
([modules state]
(->> modules
(map #(wrap-output % state))
(into [])
)))
(defn merge-init-fn [module init-fn]
(-> module
(update :entries util/vec-conj (output/ns-only init-fn))
(update :append-js str (output/fn-call init-fn))))
(defn rewrite-modules
"rewrites :modules to add browser related things"
[{:keys [worker-info] :as state} mode {:keys [modules module-loader] :as config}]
(let [default-module (pick-default-module-from-config modules)]
(reduce-kv
(fn [mods module-id {:keys [web-worker init-fn] :as module-config}]
(let [default?
(= default-module module-id)
module-config
(-> module-config
(assoc :force-append true
:default default?)
(cond->
init-fn
(merge-init-fn init-fn)
;; REPL client - only for watch (via worker-info), not compile
(and default? (= :dev mode) worker-info)
(inject-repl-client state config)
;; DEVTOOLS console, it is prepended so it loads first in case anything wants to log
(and default? (= :dev mode))
(inject-devtools-console state config)
(and worker-info (not web-worker) (not (false? (get-in config [:devtools :enabled]))))
(update :append-js str "\nshadow.cljs.devtools.client.browser.module_loaded('" (name module-id) "');\n")
;; MODULE-LOADER
;; default module brings in shadow.loader
(and module-loader default?)
(update :entries #(into '[shadow.loader] %))
;; other modules just need to tell the loader they finished loading
(and module-loader (not (or default? web-worker)))
(update :append-js str "\nshadow.loader.set_loaded('" (name module-id) "');")
(= :dev mode)
(inject-preloads state config)
))]
(assoc mods module-id module-config)))
{}
modules)))
(defn configure-modules
[{:keys [worker-info] :as state} mode {:keys [modules module-loader] :as config}]
(let [modules (rewrite-modules state mode config)]
(build-api/configure-modules state modules)))
(defn configure
[state mode config]
(let [{:keys [output-dir asset-path public-dir public-path modules] :as config}
(-> (merge default-browser-config config)
(cond->
(not (false? (get-in config [:devtools :autoload])))
(assoc-in [:devtools :autoload] true)))
output-wrapper?
(let [x (get-in state [:compiler-options :output-wrapper])]
(if (false? x)
false
(or x (and (= :release mode) (= 1 (count modules))))))]
(-> state
(assoc ::build/config config) ;; so the merged defaults don't get lost
(assoc-in [:compiler-options :output-wrapper] output-wrapper?)
(cond->
;; only turn this on with 2+ modules, not required for single file
(and output-wrapper?
(not= 1 (count modules))
(not (seq (get-in state [:compiler-options :rename-prefix-namespace]))))
(assoc-in [:compiler-options :rename-prefix-namespace] "$APP")
asset-path
(build-api/merge-build-options {:asset-path asset-path})
output-dir
(build-api/merge-build-options {:output-dir (io/file output-dir)})
;; backwards compatibility so it doesn't break existing configs
public-dir
(build-api/merge-build-options {:output-dir (io/file public-dir)})
public-path
(build-api/merge-build-options {:asset-path public-path})
(not (contains? (:js-options config) :js-provider))
(build-api/with-js-options {:js-provider :shadow})
(and (= :dev mode) (:worker-info state))
(-> (repl/setup)
(shared/merge-repl-defines config)))
(configure-modules mode config)
)))
(defn flush-manifest
[{:keys [build-options] :as state} include-foreign?]
(let [data
(->> (or (::closure/modules state)
(:build-modules state))
(map (fn [{:keys [module-id output-name entries depends-on sources foreign-files] :as mod}]
{:module-id module-id
:name module-id
:output-name output-name
:entries entries
:depends-on depends-on
:sources
(->> sources
(map #(get-in state [:sources %]))
(map :resource-name)
(into []))}))
(into []))
manifest-name
(:manifest-name build-options "manifest.edn")
manifest-file
(data/output-file state manifest-name)
manifest
(cond
(str/ends-with? manifest-name ".edn")
(core-ext/safe-pr-str data)
(str/ends-with? manifest-name ".json")
(with-out-str
(json/pprint data :escape-slash false))
:else
(throw (ex-info (format "invalid manifest output format: %s" manifest-name) {:manifest-name manifest-name})))]
(spit manifest-file manifest))
state)
(defn flush-module-data [state]
(let [module-data
(module-loader-data state)
json-file
(data/output-file state "module-loader.json")
edn-file
(data/output-file state "module-loader.edn")]
(io/make-parents json-file)
(spit json-file (json module-data))
(spit edn-file (core-ext/safe-pr-str module-data))
state
))
(defn hash-optimized-module [{:keys [sources prepend append output output-name] :as mod} state module-hash-names]
(let [signature
(->> sources
(map #(data/get-source-by-id state %))
;; these are prepended after closure compilation
;; so they need to be included in the hash calculation
;; not just output
(filter #(= :shadow-js (:type %)))
(map #(data/get-output! state %))
(map :js)
(into [prepend output append])
(remove nil?)
(util/md5hex-seq))
signature
(cond
(true? module-hash-names)
signature
(and (number? module-hash-names)
(<= 0 module-hash-names 32))
(subs signature 0 module-hash-names)
:else
(throw (ex-info (format "invalid :module-hash-names value %s" module-hash-names)
{:tag ::module-hash-names
:module-hash-names module-hash-names})))]
(assoc mod :output-name (str/replace output-name #".js$" (str "." signature ".js")))))
(defn hash-optimized-modules [state module-hash-names]
(update state ::closure/modules
(fn [optimized]
(->> optimized
(map #(hash-optimized-module % state module-hash-names))
(into [])))))
(defn inject-loader-setup-dev
[state config]
(let [{:keys [module-uris module-infos]}
(module-loader-data state)]
(update-in state [:build-modules 0 :prepend]
str "\nvar shadow$loader = " (json {:uris module-uris :infos module-infos}) ";\n")
))
;; in release just append to first (base) module
(defn inject-loader-setup-release
[state {:keys [module-loader module-hash-names] :as config}]
(let [{:keys [module-uris module-infos] :as md} (module-loader-data state)]
(update-in state [::closure/modules 0]
(fn [{:keys [module-id] :as mod}]
;; since prepending this text changes the md5 of the output
;; we need to re-hash that module again
(-> mod
(update :prepend str "\nvar shadow$loader = " (json {:uris module-uris :infos module-infos}) ";\n")
(cond->
module-hash-names
;; previous hash already changed the output-name, reset it back
(-> (assoc :output-name (str (name module-id) ".js"))
(hash-optimized-module state module-hash-names))))))))
(defn get-all-module-deps [{:keys [build-modules] :as state} {:keys [depends-on] :as mod}]
;; FIXME: not exactly pretty, just abusing the fact that build-modules is already ordered
(->> (reverse (or (get-in state [:shadow.build.closure/modules])
(get-in state [:build-modules])))
(remove :dead)
(reduce
(fn [{:keys [deps order] :as x} {:keys [module-id] :as mod}]
(if-not (contains? deps module-id)
x
{:deps (set/union deps (:depends-on mod))
:order (conj order mod)}))
{:deps depends-on
:order []})
(:order)
(reverse)
(into [])))
(defn flush-unoptimized-module!
[{:keys [unoptimizable build-options] :as state}
{:keys [goog-base output-name prepend append sources web-worker] :as mod}
target]
(let [{:keys [dev-inline-js]}
build-options
sources
(if-not web-worker
sources
(let [mods (get-all-module-deps state mod)]
(-> []
(into (mapcat :sources) mods)
(into sources))))
inlineable-sources
(if-not dev-inline-js
[]
(->> sources
(map #(data/get-source-by-id state %))
(filter output/inlineable?)
(into [])))
inlineable-set
(into #{} (map :resource-id) inlineable-sources)
inlined-js
(->> inlineable-sources
(map #(data/get-output! state %))
(map :js)
(str/join "\n"))
;; goog.writeScript_ (via goog.require) will set these
;; since we skip these any later goog.require (that is not under our control, ie REPL)
;; won't recognize them as loaded and load again
closure-require-hack
(->> inlineable-sources
(map :output-name)
(map (fn [output-name]
;; not entirely sure why we are setting the full path and just the name
;; goog seems to do that
(str "SHADOW_ENV.setLoaded(\"" + output-name "\");")
))
(str/join "\n"))
require-files
(->> sources
(remove inlineable-set)
(remove #{output/goog-base-id})
(map #(data/get-source-by-id state %))
(map :output-name)
(into []))
out
(str inlined-js
prepend
closure-require-hack
(str "SHADOW_ENV.load({}, " (json/write-str require-files) ");\n")
append)
out
(if (or goog-base web-worker)
(str unoptimizable
;; always include this in dev builds
;; a build may not include any shadow-js initially
;; but load some from the REPL later
"var shadow$provide = {};\n"
(when (and web-worker (get-in state [::build/config :module-loader]))
"var shadow$loader = false;\n")
(let [{:keys [polyfill-js]} state]
(when (and goog-base (seq polyfill-js))
(str "\n" polyfill-js)))
(-> state
(cond->
web-worker
(assoc-in [:compiler-options :closure-defines "shadow.cljs.devtools.client.env.enabled"] false))
(output/closure-defines-and-base))
(if web-worker
(slurp (io/resource "shadow/boot/worker.js"))
(slurp (io/resource "shadow/boot/browser.js")))
"\n\n"
;; create the $CLJS var so devtools can always use it
;; always exists for :module-format :js
"goog.global[\"$CLJS\"] = goog.global;\n"
"\n\n"
out)
;; else
out)]
(spit target out)))
(defn flush-unoptimized!
[{:keys [build-modules] :as state}]
;; FIXME: this always flushes
;; it could do partial flushes when nothing was actually compiled
;; a change in :closure-defines won't trigger a recompile
;; so just checking if nothing was compiled is not reliable enough
;; flushing really isn't that expensive so just do it always
(when-not (seq build-modules)
(throw (ex-info "flush before compile?" {})))
(output/flush-sources state)
(util/with-logged-time
[state {:type :flush-unoptimized}]
(doseq [{:keys [output-name] :as mod} build-modules]
(flush-unoptimized-module! state mod (data/output-file state output-name)))
state
))
(defn flush-unoptimized
[state]
"util for ->"
(flush-unoptimized! state)
state)
(defn flush [state mode {:keys [module-loader module-hash-names] :as config}]
(case mode
:dev
(-> state
(cond->
module-loader
(-> (inject-loader-setup-dev config)
(flush-module-data)))
(flush-unoptimized)
(flush-manifest false))
:release
(-> state
;; must hash before adding loader since it needs to know the final uris of the modules
;; it will change the uri of the base module after
(cond->
module-hash-names
(hash-optimized-modules module-hash-names)
;; true to inject the loader data (which changes the signature)
;; any other true-ish value still generates the module-loader.edn data files
;; but does not inject (ie. change the signature)
(true? module-loader)
(inject-loader-setup-release config)
(get-in state [:compiler-options :output-wrapper])
(apply-output-wrapper))
(output/flush-optimized)
(cond->
module-loader
(flush-module-data))
(flush-manifest true))))
(defn make-web-worker-prepend [state mod]
(let [all
(get-all-module-deps state mod)
import-script-names
(->> all
(map :output-name)
(map pr-str)
(str/join ","))]
(str "importScripts(" import-script-names ");")))
(def imul-js-fix
(str/trim (slurp (io/resource "cljs/imul.js"))))
(defn module-wrap
"add web specific prepends to each module"
;; FIXME: node environments should not require the Math.imul fix right?
[{::build/keys [mode] :as state}]
(update state :build-modules
(fn [modules]
(->> modules
(map (fn [{:keys [goog-base web-worker] :as mod}]
(-> mod
(cond->
goog-base
(update :prepend str imul-js-fix "\n")
(and (= :release mode) web-worker)
(update :prepend str (make-web-worker-prepend state mod) "\n")
))))
(into [])
))))
(defn bootstrap-host-build? [{:keys [sym->id] :as state}]
(contains? sym->id 'shadow.cljs.bootstrap.env))
(defn bootstrap-host-info [state]
(reduce
(fn [state {:keys [module-id sources] :as mod}]
(let [all-provides
(->> sources
(map #(data/get-source-by-id state %))
(map :provides)
(reduce set/union))
load-str
(str "shadow.cljs.bootstrap.env.set_loaded(" (json/write-str all-provides) ");")]
(update-in state [:output [:shadow.build.modules/append module-id] :js] str "\n" load-str "\n")
))
state
(:build-modules state)))
(defn maybe-inject-cljs-loader-constants
[{:keys [sym->id] :as state} mode config]
(if-not (contains? sym->id 'cljs.loader)
state
(let [{:keys [module-uris module-infos] :as data}
(module-loader-data state)]
(assoc state :loader-constants {'cljs.core/MODULE_URIS module-uris
'cljs.core/MODULE_INFOS module-infos})
)))
(defmethod build-log/event->str ::npm-version-check
[event]
"Checking used npm package versions")
(defmethod build-log/event->str ::npm-version-conflict
[{:keys [package-name wanted-dep wanted-version installed-version] :as event}]
(format "npm package \"%s\" expected version \"%s@%s\" but \"%s\" is installed."
package-name
wanted-dep
wanted-version
installed-version))
(defn check-npm-versions [{::keys [version-checked] :keys [npm] :as state}]
(let [pkg-index
(->> (data/get-build-sources state)
(filter #(= :shadow-js (:type %)))
(map :package-name)
(remove nil?)
(remove #(contains? version-checked %))
(into #{})
(reduce
(fn [m package-name]
(assoc m package-name (npm/find-package npm package-name)))
{}))]
(if-not (seq pkg-index)
;; prevent the extra verbose log entry when no check is done
state
;; keeping track of what we checked so its not repeatedly check during watch
;; FIXME: updating npm package while watch is running will not check again
(util/with-logged-time [state {:type ::npm-version-check}]
(reduce
(fn [state package-name]
(doseq [[dep wanted-version]
(merge (get-in pkg-index [package-name :package-json "dependencies"])
(get-in pkg-index [package-name :package-json "peerDependencies"]))
;; not all deps end up being used so we don't need to check the version
:when (get pkg-index dep)
:let [installed-version (get-in pkg-index [dep :package-json "version"])]
:when (not (npm-deps/semver-intersects wanted-version installed-version))]
(util/warn state {:type ::npm-version-conflict
:package-name package-name
:wanted-dep dep
:wanted-version wanted-version
:installed-version installed-version}))
(update state ::version-checked util/set-conj package-name))
state
(keys pkg-index))))))
(defn process
[{::build/keys [stage mode config] :as state}]
(case stage
:configure
(configure state mode config)
:compile-prepare
(maybe-inject-cljs-loader-constants state mode config)
:compile-finish
(-> state
(module-wrap)
(check-npm-versions)
(cond->
(bootstrap-host-build? state)
(bootstrap-host-info)))
:optimize-prepare
(-> state
(assoc-in [:compiler-options :closure-defines 'shadow.js.process.browser] true))
:flush
(flush state mode config)
state
))
(comment
(shadow.cljs.devtools.api/compile :browser)
(shadow.cljs.devtools.api/release :browser)
(shadow.cljs.devtools.api/watch :browser {:verbose true}))