WIP: clj-kondo hooks for require-python

.github/workflows/test.yml

@@ -10,6 +10,25 @@ on:
   pull_request:
 
 jobs:
+  clj-kondo-hooks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Clojure
+        uses: DeLaGuardo/setup-clojure@11.0
+        with:
+          cli: 1.11.1.1347
+
+      - name: Install clj-kondo
+        run: |
+          curl -sLO https://raw.githubusercontent.com/clj-kondo/clj-kondo/master/script/install-clj-kondo
+          chmod +x install-clj-kondo
+          sudo ./install-clj-kondo
+
+      - name: Test clj-kondo hooks
+        run: clojure -M:clj-kondo-test
+
   unit-test:
     runs-on: ${{matrix.os}}
     strategy:

deps.edn

@@ -71,4 +71,12 @@
             :exec-args {:installer :local
                         :artifact "target/libpython-clj.jar"}}
 
+           :clj-kondo-test
+           {:extra-deps {com.cognitect/test-runner
+                         {:git/url "https://github.com/cognitect-labs/test-runner"
+                          :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}}
+            :extra-paths ["test"]
+            :main-opts ["-m" "cognitect.test-runner"
+                        "-n" "clj-kondo.hook-test"]}
+
            }}

resources/clj-kondo.exports/clj-python/libpython-clj/config.edn

@@ -1,13 +1,59 @@
-{:hooks
+{:lint-as {libpython-clj2.python/with clojure.core/with-open}
+
+ :hooks
  {:analyze-call {libpython-clj.jna.base/def-pylib-fn
                  hooks.libpython-clj.jna.base.def-pylib-fn/def-pylib-fn
+
                  libpython-clj.require/import-python
-                 hooks.libpython-clj.require.import-python/import-python}}
-  :linters {:unused-namespace {:exclude [builtins.list
-                                         builtins.dict
-                                         builtins.set
-                                         builtins.tuple
-                                         builtins.frozenset
-                                         builtins.str
-                                         builtins]}}
-}
+                 hooks.libpython-clj.require.import-python/import-python
+                 libpython-clj2.require/import-python
+                 hooks.libpython-clj.require.import-python/import-python
+
+                 libpython-clj.require/require-python
+                 hooks.libpython-clj.require.require-python/require-python
+                 libpython-clj2.require/require-python
+                 hooks.libpython-clj.require.require-python/require-python
+
+                 libpython-clj2.python/py.
+                 hooks.libpython-clj.python.py-macros/py-macro
+                 libpython-clj2.python/py..
+                 hooks.libpython-clj.python.py-macros/py-macro
+                 libpython-clj2.python/py.-
+                 hooks.libpython-clj.python.py-macros/py-macro
+                 libpython-clj2.python/py*
+                 hooks.libpython-clj.python.py-macros/py-macro
+                 libpython-clj2.python/py**
+                 hooks.libpython-clj.python.py-macros/py-macro}}
+
+ :linters
+ {:unresolved-namespace
+  {:exclude [python
+             python.list
+             python.dict
+             python.set
+             python.tuple
+             python.frozenset
+             python.str
+             builtins
+             builtins.list
+             builtins.dict
+             builtins.set
+             builtins.tuple
+             builtins.frozenset
+             builtins.str]}
+
+  :unused-namespace
+  {:exclude [builtins.list
+             builtins.dict
+             builtins.set
+             builtins.tuple
+             builtins.frozenset
+             builtins.str
+             builtins
+             python
+             python.list
+             python.dict
+             python.set
+             python.tuple
+             python.frozenset
+             python.str]}}}

resources/clj-kondo.exports/clj-python/libpython-clj/hooks/libpython_clj/python/py_macros.clj

@@ -0,0 +1,16 @@
+(ns hooks.libpython-clj.python.py-macros
+  (:require [clj-kondo.hooks-api :as api]))
+
+(defn py-macro
+  "Transform py macros to just evaluate the object, ignoring method/attribute symbols.
+   (py. obj method arg) -> obj
+   (py.. obj method1 method2) -> obj
+   (py.- obj attr) -> obj
+   (py* callable args) -> callable
+   (py** callable args kwargs) -> callable"
+  [{:keys [node]}]
+  (let [children (:children node)
+        obj-node (second children)]
+    (if obj-node
+      {:node obj-node}
+      {:node (api/token-node nil)})))

resources/clj-kondo.exports/clj-python/libpython-clj/hooks/libpython_clj/require/require_python.clj

@@ -0,0 +1,106 @@
+(ns hooks.libpython-clj.require.require-python
+  (:require [clj-kondo.hooks-api :as api]))
+
+(def ^:private libpython-unary-flags
+  #{:bind-ns :reload :no-arglists})
+
+(def ^:private libpython-binary-flags
+  #{:refer})
+
+(defn- get-sexpr [node]
+  (when node
+    (try
+      (api/sexpr node)
+      (catch Exception _ nil))))
+
+(defn- quoted-form?
+  [sexpr]
+  (and (seq? sexpr)
+       (= 'quote (first sexpr))))
+
+(defn- unquote-sexpr
+  [sexpr]
+  (if (quoted-form? sexpr)
+    (second sexpr)
+    sexpr))
+
+(defn- extract-bind-ns-var
+  [spec-data]
+  (let [pairs (partition 2 (rest spec-data))
+        bind-ns-val (some (fn [[k v]] (when (= :bind-ns k) v)) pairs)
+        as-val (some (fn [[k v]] (when (= :as k) v)) pairs)]
+    (when bind-ns-val
+      (or as-val
+          (when-let [first-sym (first spec-data)]
+            (symbol (last (clojure.string/split (str first-sym) #"\."))))))))
+
+(defn- extract-refer-symbols
+  [spec-data]
+  (let [pairs (partition 2 (rest spec-data))
+        refer-val (some (fn [[k v]] (when (= :refer k) v)) pairs)]
+    (when (and refer-val (vector? refer-val))
+      refer-val)))
+
+(defn- filter-spec-data
+  [spec-data]
+  (loop [result []
+         remaining (rest spec-data)]
+    (if (empty? remaining)
+      (vec (cons (first spec-data) result))
+      (let [item (first remaining)
+            next-item (second remaining)]
+        (cond
+          (libpython-unary-flags item)
+          (if (boolean? next-item)
+            (recur result (drop 2 remaining))
+            (recur result (rest remaining)))
+
+          (libpython-binary-flags item)
+          (recur result (drop 2 remaining))
+
+          :else
+          (recur (conj result item) (rest remaining)))))))
+
+(defn- process-spec-data
+  [sexpr]
+  (let [unquoted (unquote-sexpr sexpr)]
+    (cond
+      (vector? unquoted)
+      {:spec-data (filter-spec-data unquoted)
+       :bind-ns-var (extract-bind-ns-var unquoted)
+       :refer-symbols (extract-refer-symbols unquoted)}
+
+      (symbol? unquoted)
+      {:spec-data unquoted
+       :bind-ns-var nil
+       :refer-symbols nil}
+
+      :else
+      {:spec-data unquoted
+       :bind-ns-var nil
+       :refer-symbols nil})))
+
+(defn- make-require-form
+  [specs]
+  (list* 'require
+         (map (fn [spec] (list 'quote spec)) specs)))
+
+(defn- make-def-form
+  [var-name]
+  (list 'def var-name nil))
+
+(defn require-python
+  [{:keys [node]}]
+  (let [form (get-sexpr node)
+        args (rest form)
+        processed (map process-spec-data args)
+        spec-data-list (map :spec-data processed)
+        bind-ns-vars (filter some? (map :bind-ns-var processed))
+        refer-symbols (mapcat :refer-symbols processed)
+        require-form (make-require-form spec-data-list)
+        def-forms (map make-def-form (concat bind-ns-vars refer-symbols))
+        result-form (if (seq def-forms)
+                      (list* 'do require-form def-forms)
+                      require-form)
+        result-node (api/coerce result-form)]
+    {:node (with-meta result-node (meta node))}))

test/clj_kondo/fixtures/py_dot_test.clj

@@ -0,0 +1,17 @@
+(ns clj-kondo.fixtures.py-dot-test
+  (:require [libpython-clj2.python :as py :refer [py. py.. py.-]]))
+
+(defn test-py-macros [obj serialization padding hashes Fernet rsa]
+  (py.. Fernet generate_key decode)
+  (py.. serialization -Encoding -PEM)
+  (py.. obj (private_bytes
+             :encoding (py.. serialization -Encoding -PEM)
+             :format (py.. serialization -PrivateFormat -PKCS8))
+        decode)
+  (py.. padding (OAEP
+                 :mgf (py.. padding (MGF1 :algorithm (py.. hashes SHA256)))
+                 :algorithm (py.. hashes SHA1)
+                 :label nil))
+  (py.. rsa (generate_private_key :public_exponent 65537 :key_size 2048))
+  (py. obj method arg1 arg2)
+  (py.- obj some_attribute))

test/clj_kondo/fixtures/py_with_test.clj

@@ -0,0 +1,6 @@
+(ns clj-kondo.fixtures.py-with-test
+  (:require [libpython-clj2.python :as py]))
+
+(defn test-py-with [testcode]
+  (py/with [f (py/call-attr testcode "FileWrapper" "content")]
+    (py/call-attr f "read")))

test/clj_kondo/fixtures/require_python_edge_cases.clj

@@ -0,0 +1,18 @@
+(ns clj-kondo.fixtures.require-python-edge-cases
+  "Edge case tests for require-python clj-kondo hook."
+  (:require [libpython-clj2.require :refer [require-python]]))
+
+(require-python '[json :refer :all])
+
+(require-python '[collections :refer :*])
+
+(require-python '[datetime :bind-ns true :reload true :no-arglists true])
+
+(require-python '[urllib.parse :as parse :bind-ns true :refer [urlencode urlparse]])
+
+(defn test-refer-all []
+  (datetime/now))
+
+(defn test-combined-refer []
+  (urlencode {})
+  (urlparse "http://example.com"))

test/clj_kondo/fixtures/require_python_test.clj

@@ -0,0 +1,49 @@
+(ns clj-kondo.fixtures.require-python-test
+  "Test fixtures for require-python clj-kondo hook.
+   This file should produce ZERO errors when linted with the hook."
+  (:require [libpython-clj2.require :refer [require-python import-python]]))
+
+(import-python)
+
+(require-python '[numpy :as np]
+                '[pandas :as pd]
+                '[matplotlib.pyplot :as plt])
+
+(require-python '[pathlib :bind-ns true])
+
+(require-python '[requests :reload true])
+
+(require-python '[sklearn.linear_model :no-arglists true])
+
+(require-python '[pywebpush :bind-ns true :refer [webpush]])
+
+(require-python '[werkzeug.utils :refer [secure_filename]])
+
+(require-python '[google.cloud.secretmanager :as secretmanager :bind-ns true])
+
+(require-python 'operator
+                'base64
+                'socket)
+
+(require-python '[cryptography.fernet.Fernet :as Fernet :bind-ns true]
+                '[cryptography.hazmat.primitives.hashes :as hashes :bind-ns true])
+
+(require-python '[os :reload])
+
+(defn test-bind-ns-usage []
+  (pathlib/Path "/tmp")
+  (Fernet/generate_key))
+
+(defn test-refer-usage []
+  (webpush {})
+  (secure_filename "test.txt"))
+
+(defn test-alias-usage []
+  (np/array [1 2 3])
+  (pd/DataFrame {})
+  (secretmanager/SecretManagerServiceClient))
+
+(defn test-python-builtins []
+  (python/len [1 2 3])
+  (python.list/append [] 1)
+  (python.dict/keys {}))

test/clj_kondo/hook_test.clj

@@ -0,0 +1,54 @@
+(ns clj-kondo.hook-test
+  (:require [clojure.test :refer [deftest testing is]]
+            [clojure.java.shell :refer [sh]]
+            [clojure.string :as str]))
+
+(def ^:private config-dir
+  "resources/clj-kondo.exports/clj-python/libpython-clj")
+
+(def ^:private fixtures-dir
+  "test/clj_kondo/fixtures")
+
+(defn- run-clj-kondo [file]
+  (sh "clj-kondo" "--lint" file "--config-dir" config-dir))
+
+(defn- has-require-python-errors? [output]
+  (boolean (re-find #"(Unknown require option|:bind-ns|:reload|:no-arglists)" output)))
+
+(defn- has-unresolved-refer-errors? [output]
+  (boolean (re-find #"Unresolved symbol: (webpush|secure_filename|urlencode|urlparse)" output)))
+
+(deftest require-python-hook-test
+  (testing "require_python_test.clj - basic require-python usage"
+    (let [{:keys [out err]} (run-clj-kondo (str fixtures-dir "/require_python_test.clj"))
+          output (str out err)]
+      (is (not (has-require-python-errors? output))
+          (str "Found require-python errors in output:\n" output))
+      (is (not (has-unresolved-refer-errors? output))
+          (str "Found unresolved symbol errors for referred symbols:\n" output)))))
+
+(deftest require-python-edge-cases-test
+  (testing "require_python_edge_cases.clj - edge cases and variations"
+    (let [{:keys [out err]} (run-clj-kondo (str fixtures-dir "/require_python_edge_cases.clj"))
+          output (str out err)]
+      (is (not (has-require-python-errors? output))
+          (str "Found require-python errors in output:\n" output))
+      (is (not (has-unresolved-refer-errors? output))
+          (str "Found unresolved symbol errors for referred symbols:\n" output)))))
+
+(defn- has-py-dot-errors? [output]
+  (boolean (re-find #"(Unresolved symbol: py\.|unresolved var.*py\.)" output)))
+
+(deftest py-dot-hook-test
+  (testing "py_dot_test.clj - py. and py.. macros"
+    (let [{:keys [out err]} (run-clj-kondo (str fixtures-dir "/py_dot_test.clj"))
+          output (str out err)]
+      (is (not (has-py-dot-errors? output))
+          (str "Found py. related errors in output:\n" output)))))
+
+(deftest py-with-test
+  (testing "py_with_test.clj - py/with binding"
+    (let [{:keys [out err]} (run-clj-kondo (str fixtures-dir "/py_with_test.clj"))
+          output (str out err)]
+      (is (not (re-find #"Unresolved symbol: f" output))
+          (str "py/with binding not recognized:\n" output)))))