Skip to content

Commit

Permalink
Issue 25 - data driven require.clj pathway (#28)
Browse files Browse the repository at this point in the history 
* light overhaul; tests added

* whitespace

* formatting
  • Loading branch information
@jjtolton @cnuernber
jjtolton authored and cnuernber committed Dec 15, 2019
1 parent 70c2ae8 commit b2c4ba7
Show file tree
Hide file tree
Showing 2 changed files with 219 additions and 92 deletions.
233 changes: 147 additions & 86 deletions src/libpython_clj/require.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@
(number? x) ""
:else (py-class-argspec x)))


(defn pyarglists
([argspec] (pyarglists argspec
(if-let [defaults
Expand Down Expand Up @@ -142,10 +141,6 @@
{:or or-map})
(when (not-empty as-varkw)
as-varkw))




opt-args
(cond
(and (empty? kwargs-map)
Expand All @@ -168,7 +163,6 @@
arglists
(recur argspec' defaults' arglists))))))


(defn py-fn-metadata [fn-name x {:keys [no-arglists?]}]
(let [fn-argspec (pyargspec x)
fn-docstr (get-pydoc x)]
Expand All @@ -183,7 +177,6 @@
(catch Throwable e
nil))))))


(defn ^:private load-py-fn [f fn-name fn-module-name-or-ns
options]
(let [fn-ns (symbol (str fn-module-name-or-ns))
Expand Down Expand Up @@ -215,8 +208,6 @@
results
(first items)
(rest items))


:else
;; unary flag -- add flag but scan current item/s
(next-flag-item
Expand All @@ -225,7 +216,6 @@
item
items)))


;; scan flags
(next-flag-item [supported-flags results item items]
(cond
Expand Down Expand Up @@ -259,9 +249,37 @@
(rest reqs)))]
(trampoline get-flags supported-flags reqs)))



(defn ^:private load-python-lib [req]
(defn- extract-refer-symbols
[{:keys [refer this-module]} public-data]
(cond
;; include everything into the current namespace,
;; ignore __all__ directive
(or (refer :all)
(and (not (py/has-attr? this-module "__all__"))
(refer :*)))
(keys public-data)
;; only include that specfied by __all__ attribute
;; of python module if specified, else same as :all
(refer :*)
(->> (py/get-attr this-module "__all__")
(map (fn [item-name]
(let [item-sym (symbol item-name)]
(when (contains? public-data item-sym)
item-sym))))
(remove nil?))

;; [.. :refer [..]] behavior
:else
(do
(when-let [missing (->> refer
(remove (partial contains? public-data))
seq)]
(throw (Exception.
(format "'refer' symbols not found: %s"
(vec missing)))))
refer)))

(defn ^:private python-lib-configuration [req]
(let [supported-flags #{:reload :no-arglists}
[module-name & etc] req
flags (parse-flags supported-flags etc)
Expand All @@ -286,80 +304,122 @@
current-ns-sym (symbol (str current-ns))
python-namespace (find-ns module-name-or-ns)
this-module (import-module (str module-name))]
{:supported-flags supported-flags
:etc etc
:reload? reload?
:no-arglists? no-arglists?
:module-name module-name
:module-name-or-ns module-name-or-ns
:exclude exclude
:refer refer
:current-ns current-ns
:current-ns-sym current-ns-sym
:python-namespace python-namespace
:this-module this-module}))

(defn- extract-public-data
[{:keys [exclude python-namespace module-name-or-ns]}]
(let [python-namespace
(or python-namespace
(find-ns module-name-or-ns))]
(->> (ns-publics python-namespace)
(remove #(exclude (first %)))
(into {}))))

(defn- reload-python-ns!
[module-name this-module module-name-or-ns]
(do
(remove-ns module-name)
(reload-module this-module)
(create-ns module-name-or-ns)))

(defn- create-python-ns!
[module-name-or-ns]
(create-ns module-name-or-ns))

(defn ^:private maybe-reload-or-create-ns!
[{:keys [reload?
this-module
module-name
module-name-or-ns]
python-namespace :python-namespace}]
(cond
reload? (reload-python-ns! module-name
this-module
module-name-or-ns)
(not python-namespace) (create-python-ns! module-name-or-ns)))

(defn enhanced-python-lib-configuration
[{:keys [python-namespace exclude this-module]
:as lib-config}]
(let [public-data (extract-public-data lib-config)]
(merge
lib-config
{:public-data public-data
:refer-symbols (extract-refer-symbols lib-config
public-data)})))

(defn- bind-py-symbols-to-ns!
[{:keys [reload?
python-namespace
this-module
module-name-or-ns
no-arglists?]}]
(when (or reload? (not python-namespace))
;;Mutably define the root namespace.
(doseq [[att-name v] (vars this-module)]
(try
(when v
(if (py/callable? v)
(load-py-fn v (symbol att-name) module-name-or-ns
{:no-arglists?
no-arglists?})
(intern module-name-or-ns (symbol att-name) v)))
(catch Throwable e
(log/warnf e "Failed to require symbol %s" att-name))))))

(defn- bind-module-ns!
[{:keys [current-ns-sym module-name-or-ns this-module]}]
(intern current-ns-sym
(with-meta module-name-or-ns
{:doc (doc this-module)})
this-module))

(defn- intern-public-and-refer-symbols!
[{:keys [public-data refer-symbols current-ns-sym]}]
(doseq [[s v] (select-keys public-data refer-symbols)]
(intern current-ns-sym
(with-meta s (meta v))
(deref v))))

(defn preload-python-lib! [req]
(let [lib-config (python-lib-configuration req)]
(maybe-reload-or-create-ns! lib-config)
(bind-py-symbols-to-ns! lib-config)
(bind-module-ns! lib-config)
(enhanced-python-lib-configuration lib-config)))

(cond
reload?
(do
(remove-ns module-name)
(reload-module this-module)
(create-ns module-name-or-ns))
(not python-namespace)
(create-ns module-name-or-ns))

;; bind the python module to its symbolic name
;; in the current namespace


;; create namespace for module and bind python
;; values to namespace symbols
(when (or reload?
(not python-namespace))
;;Mutably define the root namespace.
(doseq [[att-name v] (vars this-module)]
(try
(when v
(if (py/callable? v)
(load-py-fn v (symbol att-name) module-name-or-ns
{:no-arglists?
no-arglists?})
(intern module-name-or-ns (symbol att-name) v)))
(catch Throwable e
(log/warnf e "Failed to require symbol %s" att-name)))))


(let [python-namespace (find-ns module-name-or-ns)
;;ns-publics is a map of symbol to var. Var's have metadata on them.
public-data (->> (ns-publics python-namespace)
(remove #(exclude (first %)))
(into {}))]

;;Always make the loaded namespace available to the current namespace.
(intern current-ns-sym
(with-meta module-name-or-ns
{:doc (doc this-module)})
this-module)
(let [refer-symbols
(cond
;; include everything into the current namespace,
;; ignore __all__ directive
(or (refer :all)
(and (not (py/has-attr? this-module "__all__"))
(refer :*)))
(keys public-data)
;; only include that specfied by __all__ attribute
;; of python module if specified, else same as :all
(refer :*)
(->> (py/get-attr this-module "__all__")
(map (fn [item-name]
(let [item-sym (symbol item-name)]
(when (contains? public-data item-sym)
item-sym))))
(remove nil?))

;; [.. :refer [..]] behavior
:else
(do
(when-let [missing (->> refer
(remove (partial contains? public-data))
seq)]
(throw (Exception.
(format "'refer' symbols not found: %s"
(vec missing)))))
refer))]
(doseq [[s v] (select-keys public-data refer-symbols)]
(intern current-ns-sym
(with-meta s (meta v))
(deref v)))))))
(defn ^:private load-python-lib [req]
(let [{:keys [supported-flags
flags
etc
reload?
no-arglists?
module-name
module-name-or-ns
exclude
refer
current-ns
current-ns-sym
python-namespace
this-module
python-namespace
public-data
refer-symbols]
:as lib-config}
(preload-python-lib! req)]

(intern-public-and-refer-symbols! lib-config)))

(defn require-python
"## Basic usage ##
Expand Down Expand Up @@ -438,3 +498,4 @@
(load-python-lib (vector reqs))
(vector? reqs)
(load-python-lib reqs)))

78 changes: 72 additions & 6 deletions test/libpython_clj/require_python_test.clj
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
(ns libpython-clj.require-python-test
(:require [libpython-clj.require :as req :refer [require-python]]
(:require [libpython-clj.require :as req :refer [require-python]
:reload true]
[libpython-clj.python :as py]
[clojure.test :refer :all]))


;;Since this test mutates the global environment we have to accept that
;;it may not always work.
;; Since this test mutates the global environment we have to accept that
;; it may not always work.

(require-python '[math
:refer :*
:exclude [sin cos]
:as pymath])
:as pymath
:reload true])


(deftest base-require-test
Expand All @@ -22,7 +23,6 @@
(is (= pymath (py/import-module "math")))
(is (thrown? Throwable (require-python '[math :refer [blah]])))))


(deftest parse-flags-test
;; sanity check
(is (= #{:reload :foo}
Expand Down Expand Up @@ -50,3 +50,69 @@
(is (= #{:b} (parse-flags #{:a :b} '[:b true :a false])))
(is (= #{:b} (parse-flags #{:a :b} '[:b :a false])))
(is (= #{:b} (parse-flags #{:a :b} '[:b :a false :a true])))))

(deftest python-lib-configuration-test
(let [python-lib-configuration #'req/python-lib-configuration
simple-req '[csv]
simple-spec (python-lib-configuration simple-req)
{:keys [exclude
supported-flags
current-ns-sym
module-name
module-name-or-ns
reload?
no-arglists?
etc
current-ns
this-module
python-namespace
refer]} simple-spec
csv-module (py/import-module "csv")]

;; no exclusions
(is (= #{} exclude))
(is (= #{:no-arglists :reload} supported-flags))
(is (= 'libpython-clj.require-python-test
current-ns-sym
(symbol (str current-ns))))
(is (= 'csv module-name module-name-or-ns))
(is (nil? reload?))
(is (nil? no-arglists?))
(is (= {} etc))
(is (= csv-module this-module)))


(let [python-lib-configuration #'req/python-lib-configuration
simple-req '[requests
:reload true
:refer [get]
:no-arglists]
simple-spec (python-lib-configuration simple-req)
{:keys [exclude
supported-flags
current-ns-sym
module-name
module-name-or-ns
reload?
no-arglists?
etc
current-ns
this-module
python-namespace
refer]} simple-spec
requests-module (py/import-module "requests")]

;; no exclusions
(is (= #{} exclude))
(is (= #{:no-arglists :reload} supported-flags))
(is (= 'libpython-clj.require-python-test
current-ns-sym
(symbol (str current-ns))))
(is (= 'requests module-name module-name-or-ns))
(is (= reload? :reload))
(is (= no-arglists? :no-arglists))
(is (= { :refer '[get]}) etc)
(is (= requests-module this-module))
(is (= #{'get} refer))
(is (nil? python-namespace))))

0 comments on commit b2c4ba7

Please sign in to comment.