Skip to content

Commit

Permalink
CLJS-1960: Require CommonJS modules directly from a ClojureScript nam…
Browse files Browse the repository at this point in the history 
…espace

This patch addresses the first part of the solution outlined in the following
design doc:
https://github.com/clojure/clojurescript/wiki/Enhanced-Node.js-Modules-Support

It makes possible to specify, install and require Node.js dependencies directly
from ClojureScript namespaces. Future work can make it possible to support
specifying these dependencies in `deps.cljs` files and handling conflict
resolution between upstream foreign dependencies and foreign dependencies
specified directly in the compiler options.
  • Loading branch information
@anmonteiro
anmonteiro authored and dnolen committed Mar 10, 2017
1 parent 1d38f73 commit 777d41b
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 70 deletions.
57 changes: 51 additions & 6 deletions src/main/cljs/cljs/module_deps.js
View file
@@ -1,26 +1,71 @@
var path = require('path');
var mdeps = require('module-deps');
var nodeResolve = require('resolve');
var browserResolve = require('browser-resolve');

var md = mdeps({});
var deps_files = [];
var target = 'CLJS_TARGET';
var filename = path.resolve(__dirname, 'JS_FILE');
var resolver = target === 'nodejs' ? nodeResolve : browserResolve;

var md = mdeps({
resolve: function(id, parent, cb) {
// set the basedir properly so we don't try to resolve requires in the Closure
// Compiler processed `node_modules` folder.
parent.basedir = parent.filename === filename ? __dirname: path.dirname(parent.filename);

resolver(id, parent, cb);
},
filter: function(id) {
return !nodeResolve.isCore(id);
}});

var pkgJsons = [];
var deps_files = {};

md.on('package', function (pkg) {
// we don't want to include the package.json for users' projects
if (/node_modules/.test(pkg.__dirname)) {
deps_files.push({file: path.join(pkg.__dirname, 'package.json')});
var pkgJson = {
file: path.join(pkg.__dirname, 'package.json'),
};

if (pkg.name != null) {
pkgJson.provided = [ pkg.name ];
}

if (pkg.main != null) {
pkgJson.main = path.join(pkg.__dirname, pkg.main);
}

pkgJsons.push(pkgJson);
}
});

md.on('file', function(file) {
deps_files.push({file: file});
deps_files[file] = { file: file };
});

md.on('end', function() {
process.stdout.write(JSON.stringify(deps_files));
for (var i = 0; i < pkgJsons.length; i++) {
var pkgJson = pkgJsons[i];

if (deps_files[pkgJson.main] != null && pkgJson.provided != null) {
deps_files[pkgJson.main].provides = pkgJson.provided;
}

deps_files[pkgJson.file] = { file: pkgJson.file };
}

var values = [];
for (var key in deps_files) {
values.push(deps_files[key]);
}

process.stdout.write(JSON.stringify(values));
});

md.end({
file: path.resolve(path.join(__dirname, 'JS_FILE'))
file: filename,
});

md.resume();
57 changes: 4 additions & 53 deletions src/main/clojure/cljs/build/api.clj
View file
Expand Up @@ -21,11 +21,7 @@
[cljs.compiler :as comp]
[cljs.closure :as closure]
[cljs.js-deps :as js-deps])
(:import [java.io
File StringWriter
BufferedReader
Writer InputStreamReader IOException]
[java.lang ProcessBuilder]))
(:import [java.io File]))

;; =============================================================================
;; Useful Utilities
Expand Down Expand Up @@ -219,57 +215,12 @@
(binding [ana/*cljs-warning-handlers* (:warning-handlers opts ana/*cljs-warning-handlers*)]
(closure/watch source opts compiler-env stop))))

(defn- alive? [proc]
(try (.exitValue proc) false (catch IllegalThreadStateException _ true)))

(defn- pipe [^Process proc in ^Writer out]
;; we really do want system-default encoding here
(with-open [^java.io.Reader in (-> in InputStreamReader. BufferedReader.)]
(loop [buf (char-array 1024)]
(when (alive? proc)
(try
(let [len (.read in buf)]
(when-not (neg? len)
(.write out buf 0 len)
(.flush out)))
(catch IOException e
(when (and (alive? proc) (not (.contains (.getMessage e) "Stream closed")))
(.printStackTrace e *err*))))
(recur buf)))))

(defn node-module-deps
"EXPERIMENTAL: return the foreign libs entries as computed by running
the module-deps package on the supplied JavaScript entry point. Assumes
that the module-deps NPM package is either locally or globally installed."
[{:keys [file]}]
(let [code (string/replace
(slurp (io/resource "cljs/module_deps.js"))
"JS_FILE"
(string/replace file
(System/getProperty "user.dir") ""))
proc (-> (ProcessBuilder.
["node" "--eval" code])
.start)
is (.getInputStream proc)
iw (StringWriter. (* 16 1024 1024))
es (.getErrorStream proc)
ew (StringWriter. (* 1024 1024))
_ (do (.start
(Thread.
(bound-fn [] (pipe proc is iw))))
(.start
(Thread.
(bound-fn [] (pipe proc es ew)))))
err (.waitFor proc)]
(if (zero? err)
(into []
(map (fn [{:strs [file]}] file
{:file file :module-type :commonjs}))
(next (json/read-str (str iw))))
(do
(when-not (.isAlive proc)
(println (str ew)))
[]))))
[entry]
(closure/node-module-deps entry))

(comment
(node-module-deps
Expand All @@ -284,7 +235,7 @@
the module-deps package on the supplied JavaScript entry points. Assumes
that the module-deps NPM packages is either locally or globally installed."
[entries]
(into [] (distinct (mapcat node-module-deps entries))))
(closure/node-inputs entries))

(comment
(node-inputs
Expand Down
143 changes: 132 additions & 11 deletions src/main/clojure/cljs/closure.clj
View file
Expand Up @@ -47,7 +47,9 @@
[clojure.data.json :as json]
[clojure.tools.reader :as reader]
[clojure.tools.reader.reader-types :as readers])
(:import [java.io File BufferedInputStream StringWriter]
(:import [java.lang ProcessBuilder]
[java.io File BufferedInputStream BufferedReader
Writer InputStreamReader IOException StringWriter]
[java.net URL]
[java.util.logging Level]
[java.util List Random]
Expand Down Expand Up @@ -342,7 +344,6 @@
(doseq [next (seq warnings)]
(println "WARNING:" (.toString ^JSError next)))))


;; Protocols for IJavaScript and Compilable
;; ========================================

Expand All @@ -353,7 +354,7 @@
(-source-map [this] "Return the CLJS compiler generated JS source mapping"))

(extend-protocol deps/IJavaScript

String
(-foreign? [this] false)
(-closure-lib? [this] false)
Expand All @@ -362,7 +363,7 @@
(-provides [this] (:provides (deps/parse-js-ns (string/split-lines this))))
(-requires [this] (:requires (deps/parse-js-ns (string/split-lines this))))
(-source [this] this)

clojure.lang.IPersistentMap
(-foreign? [this] (:foreign this))
(-closure-lib? [this] (:closure-lib this))
Expand Down Expand Up @@ -481,7 +482,7 @@
returns a JavaScriptFile. In either case the return value satisfies
IJavaScript."
[^File file {:keys [output-file] :as opts}]
(if output-file
(if output-file
(let [out-file (io/file (util/output-directory opts) output-file)]
(compiled-file (comp/compile-file file out-file opts)))
(let [path (.getPath ^File file)]
Expand Down Expand Up @@ -567,17 +568,17 @@
(case (.getProtocol this)
"file" (-find-sources (io/file this) opts)
"jar" (find-jar-sources this opts)))

clojure.lang.PersistentList
(-compile [this opts]
(compile-form-seq [this]))
(-find-sources [this opts]
[(ana/parse-ns [this] opts)])

String
(-compile [this opts] (-compile (io/file this) opts))
(-find-sources [this opts] (-find-sources (io/file this) opts))

clojure.lang.PersistentVector
(-compile [this opts] (compile-form-seq this))
(-find-sources [this opts]
Expand Down Expand Up @@ -1339,7 +1340,7 @@

;; optimize a ClojureScript form
(optimize {:optimizations :simple} (-compile '(def x 3) {}))

;; optimize a project
(println (->> (-compile "samples/hello/src" {})
(apply add-dependencies {})
Expand Down Expand Up @@ -1730,7 +1731,7 @@
(output-deps-file opts disk-sources))))

(comment

;; output unoptimized alone
(output-unoptimized {} "goog.provide('test');\ngoog.require('cljs.core');\nalert('hello');\n")
;; output unoptimized with all dependencies
Expand Down Expand Up @@ -1916,12 +1917,19 @@
[lib])))]
(into [] (mapcat expand-lib* libs))))

(declare index-node-modules)

(defn add-implicit-options
[{:keys [optimizations output-dir]
:or {optimizations :none
output-dir "out"}
:as opts}]
(let [opts (cond-> (update opts :foreign-libs expand-libs)
(let [opts (cond-> (update opts :foreign-libs
(fn [libs]
(into []
(util/distinct-merge-by :file
(index-node-modules opts)
(expand-libs libs)))))
(:closure-defines opts)
(assoc :closure-defines
(into {}
Expand Down Expand Up @@ -1966,6 +1974,118 @@
(nil? (:closure-module-roots opts))
(assoc :closure-module-roots []))))

(defn- alive? [proc]
(try (.exitValue proc) false (catch IllegalThreadStateException _ true)))

(defn- pipe [^Process proc in ^Writer out]
;; we really do want system-default encoding here
(with-open [^java.io.Reader in (-> in InputStreamReader. BufferedReader.)]
(loop [buf (char-array 1024)]
(when (alive? proc)
(try
(let [len (.read in buf)]
(when-not (neg? len)
(.write out buf 0 len)
(.flush out)))
(catch IOException e
(when (and (alive? proc) (not (.contains (.getMessage e) "Stream closed")))
(.printStackTrace e *err*))))
(recur buf)))))

(defn maybe-install-node-deps!
[{:keys [npm-deps verbose] :as opts}]
(if-not (empty? npm-deps)
(do
(when (or ana/*verbose* verbose)
(util/debug-prn "Installing Node.js dependencies"))
(let [proc (-> (ProcessBuilder.
(into ["npm" "install" "module-deps"]
(map (fn [[dep version]] (str (name dep) "@" version)))
npm-deps))
.start)
is (.getInputStream proc)
iw (StringWriter. (* 16 1024 1024))
es (.getErrorStream proc)
ew (StringWriter. (* 1024 1024))
_ (do (.start
(Thread.
(bound-fn [] (pipe proc is iw))))
(.start
(Thread.
(bound-fn [] (pipe proc es ew)))))
err (.waitFor proc)]
(when (and (not (zero? err)) (not (.isAlive proc)))
(println (str ew)))
opts))
opts))

(defn node-module-deps
"EXPERIMENTAL: return the foreign libs entries as computed by running
the module-deps package on the supplied JavaScript entry point. Assumes
that the module-deps NPM package is either locally or globally installed."
([entry]
(node-module-deps entry
(when env/*compiler*
(:options @env/*compiler*))))
([{:keys [file]} {:keys [target] :as opts}]
(let [code (-> (slurp (io/resource "cljs/module_deps.js"))
(string/replace "JS_FILE" file)
(string/replace "CLJS_TARGET" (str "" (when target (name target)))))
proc (-> (ProcessBuilder.
["node" "--eval" code])
.start)
is (.getInputStream proc)
iw (StringWriter. (* 16 1024 1024))
es (.getErrorStream proc)
ew (StringWriter. (* 1024 1024))
_ (do (.start
(Thread.
(bound-fn [] (pipe proc is iw))))
(.start
(Thread.
(bound-fn [] (pipe proc es ew)))))
err (.waitFor proc)]
(if (zero? err)
(into []
(map (fn [{:strs [file provides]}] file
(merge
{:file file
:module-type :commonjs}
(when provides
{:provides provides}))))
(next (json/read-str (str iw))))
(do
(when-not (.isAlive proc)
(println (str ew)))
[])))))

(defn node-inputs
"EXPERIMENTAL: return the foreign libs entries as computed by running
the module-deps package on the supplied JavaScript entry points. Assumes
that the module-deps NPM packages is either locally or globally installed."
([entries]
(node-inputs entries
(when env/*compiler*
(:options @env/*compiler*))))
([entries opts]
(into [] (distinct (mapcat #(node-module-deps % opts) entries)))))

(defn index-node-modules
([]
(index-node-modules
(when env/*compiler*
(:options @env/*compiler*))))
([{:keys [npm-deps] :as opts}]
(let [node-modules (io/file "node_modules")]
(when (and (.exists node-modules) (.isDirectory node-modules))
(let [modules (map name (keys npm-deps))
deps-file (io/file (str (util/output-directory opts) File/separator
"cljs$node_modules.js"))]
(util/mkdirs deps-file)
(with-open [w (io/writer deps-file)]
(run! #(.write w (str "require('" % "');\n")) modules))
(node-inputs [{:file (.getAbsolutePath deps-file)}]))))))

(defn process-js-modules
"Given the current compiler options, converts JavaScript modules to Google
Closure modules and writes them to disk. Adds mapping from original module
Expand Down Expand Up @@ -2067,6 +2187,7 @@
(env/with-compiler-env compiler-env
(let [compiler-stats (:compiler-stats opts)
all-opts (-> opts
maybe-install-node-deps!
add-implicit-options
process-js-modules)]
(check-output-to opts)
Expand Down

0 comments on commit 777d41b

Please sign in to comment.