From 498a82bd90be0e997b296007f35ac8b0f8a39e43 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Sat, 27 Mar 2021 11:33:04 +0100 Subject: [PATCH] [#39] Tokenization of single string + add sh --- README.md | 78 +++++++++++++++++----------------- src/babashka/process.clj | 75 ++++++++++++++++++++++++-------- test/babashka/process_test.clj | 21 ++++++++- 3 files changed, 116 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 69d8bcd..a15b844 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ need it. - `$`: convenience macro around `process`. Takes command as varargs. Options can be passed via metadata on the form. Supports interpolation via `~`. +- `sh`: convenience function similar to `clojure.java.shell/sh` that sets `:out` + and `:err` to `:string` by default and blocks. Similar to `cjs/sh` it does not + check the exit code (this can be done with `check`). + - `*defaults*`: dynamic var containing overridable default options. Use `alter-var-root` to change permanently or `binding` to change temporarily. @@ -204,14 +208,6 @@ Forwarding the output of a process as the input of another process can also be d "README.md\n" ``` -`$` is a convenience macro around `process`: - -``` clojure -(def config {:output {:format :edn}}) -(-> ($ clj-kondo --config ~config --lint "src") :out slurp edn/read-string) -{:findings [], :summary {:error 0, :warning 0, :info 0, :type :summary, :duration 34}} -``` - Demo of a `cat` process to which we send input while the process is running, then close stdin and read the output of cat afterwards: @@ -238,6 +234,40 @@ then close stdin and read the output of cat afterwards: (.isAlive (:proc catp)) ;; false ``` +## $ and sh + +`$` is a convenience macro around `process`: + +``` clojure +(def config {:output {:format :edn}}) +(-> ($ clj-kondo --config ~config --lint "src") deref :out slurp edn/read-string) +{:findings [], :summary {:error 0, :warning 0, :info 0, :type :summary, :duration 34}} +``` + +`sh` is a convenience function around `process` which sets `:out` and `:err` to +`:string` and blocks automatically, similar to `clojure.java.shell/sh` and unlike `$`: + +``` clojure +(def config {:output {:format :edn}}) +(-> (sh ["clj-kondo" "--config" config" "--lint" "src"]) :out slurp edn/read-string) +{:findings [], :summary {:error 0, :warning 0, :info 0, :type :summary, :duration 34}} +``` + +## Tokenization + +Both `process`, `$` and `sh` support tokenization when passed a single string argument: + +``` clojure +(-> (sh "echo hello there") :out) +"hello there\n" +``` + +``` clojure +(-> (sh "clj-kondo --lint -" {:in "(inc)"}) :out print) +:1:1: error: clojure.core/inc is called with 0 args but expects 1 +linting took 11ms, errors: 1, warnings: 0 +``` + ## Output buffering Note that `check` will wait for the process to end in order to check the exit @@ -268,7 +298,6 @@ variables in the current environment you can proceed as follow: :env (assoc (into {} (System/getenv)) "FOO" "BAR") ``` - ## Pipelines The `pipeline` function returns a @@ -358,37 +387,6 @@ Another solution is to let bash handle the pipes by shelling out with `bash -c`. ## Notes -### Clj-kondo hook - -To make clj-kondo understand the dollar-sign macro, you can use the following config + hook code: - -`config.edn`: -``` clojure -{:hooks {:analyze-call {babashka.process/$ hooks.dollar/$}}} -``` - -`hooks/dollar.clj`: -``` clojure -(ns hooks.dollar - (:require [clj-kondo.hooks-api :as api])) - -(defn $ [{:keys [:node]}] - (let [children (doall (keep (fn [child] - (let [s (api/sexpr child)] - (when (and (seq? s) - (= 'unquote (first s))) - (first (:children child))))) - (:children node)))] - {:node (assoc node :children children)})) -``` - -Alternatively, you can either use string arguments or suppress unresolved -symbols using the following config: - -``` clojure -{:linters {:unresolved-symbol {:exclude [(babashka.process/$)]}}} -``` - ### Script termination Because `process` spawns threads for non-blocking I/O, you might have to run diff --git a/src/babashka/process.clj b/src/babashka/process.clj index dc31b28..4d8087f 100644 --- a/src/babashka/process.clj +++ b/src/babashka/process.clj @@ -1,13 +1,56 @@ (ns babashka.process (:require [clojure.java.io :as io] - [clojure.string :as str]) - (:import [java.lang ProcessBuilder$Redirect])) + [clojure.string :as str])) (ns-unmap *ns* 'Process) (ns-unmap *ns* 'ProcessBuilder) (set! *warn-on-reflection* true) +(defn tokenize + "Tokenize string to list of individual space separated arguments. + If argument contains space you can wrap it with `'` or `\"`." + [s] + (loop [s (java.io.StringReader. s) + in-double-quotes? false + in-single-quotes? false + buf (java.io.StringWriter.) + parsed []] + (let [c (.read s)] + (cond + (= -1 c) (if (pos? (count (str buf))) + (conj parsed (str buf)) + parsed) + (= 39 c) ;; single-quotes + (if in-single-quotes? + ;; exit single-quoted string + (recur s in-double-quotes? false (java.io.StringWriter.) (conj parsed (str buf))) + ;; enter single-quoted string + (recur s in-double-quotes? true buf parsed)) + + (= 92 c) ;; assume escaped quote + (let [escaped (.read s) + buf (doto buf (.write escaped))] + (recur s in-double-quotes? in-single-quotes? buf parsed)) + + (and (not in-single-quotes?) (= 34 c)) ;; double quote + (if in-double-quotes? + ;; exit double-quoted string + (recur s false in-single-quotes? (java.io.StringWriter.) (conj parsed (str buf))) + ;; enter double-quoted string + (recur s true in-single-quotes? buf parsed)) + + (and (not in-double-quotes?) + (not in-single-quotes?) + (Character/isWhitespace c)) + (recur s in-double-quotes? in-single-quotes? (java.io.StringWriter.) + (let [bs (str buf)] + (cond-> parsed + (not (str/blank? bs)) (conj bs)))) + :else (do + (.write buf c) + (recur s in-double-quotes? in-single-quotes? buf parsed)))))) + (defn- as-string-map "Helper to coerce a Clojure map with keyword keys into something coerceable to Map @@ -160,6 +203,9 @@ :err :err-enc :shutdown :inherit]} opts in (or in (:out prev)) + cmd (if (string? cmd) + (tokenize cmd) + cmd) ^java.lang.ProcessBuilder pb (if (instance? java.lang.ProcessBuilder cmd) cmd @@ -249,27 +295,22 @@ (defmacro $ [& args] (let [opts (meta &form) + args (if (and (= 1 (count args)) + (string? (first args))) + (tokenize (first args)) + args) cmd (mapv format-arg args)] `(let [cmd# ~cmd [prev# cmd#] (if-let [p# (first cmd#)] - (if #_(instance? Process p#) (:proc p#) ;; workaround for sci#432 + (if (:proc p#) [p# (rest cmd#)] [nil cmd#]) - [nil cmd#]) - #_#_[opts# cmd#] - (if-let [o# (first cmd#)] - (if (map? o#) - [o# (rest cmd#)] - [nil cmd#]) [nil cmd#])] (process prev# cmd# ~opts)))) -#_(defmacro my-foo [env] - (with-meta '($ bash -c "echo $FOO") - {:env env})) - -;; user=> (def x 10) -;; #'user/x -;; user=> (-> (my-foo {:FOO x}) :out slurp) -;; "10\n" +(defn sh + ([cmd] (sh cmd nil)) + ([cmd opts] + @(process cmd (merge {:out :string + :err :string} opts)))) diff --git a/test/babashka/process_test.clj b/test/babashka/process_test.clj index e1fc611..c64a3c7 100644 --- a/test/babashka/process_test.clj +++ b/test/babashka/process_test.clj @@ -1,9 +1,19 @@ (ns babashka.process-test - (:require [babashka.process :refer [process check $ pb start] :as p] + (:require [babashka.process :refer [tokenize process check sh $ pb start] :as p] [clojure.java.io :as io] [clojure.string :as str] [clojure.test :as t :refer [deftest is testing]])) +(deftest tokenize-test + (is (= [] (tokenize ""))) + (is (= ["hello"] (tokenize "hello"))) + (is (= ["hello" "world"] (tokenize " hello world "))) + (is (= ["foo bar" "a" "b" "c" "the d"] (tokenize "\"foo bar\" a b c \"the d\""))) + (is (= ["foo \" bar" "a" "b" "c" "the d"] (tokenize "\"foo \\\" bar\" a b c \"the d\""))) + (is (= ["echo" "foo bar"] (tokenize "echo 'foo bar'"))) + (is (= ["echo" "{\"AccessKeyId\":\"****\",\"SecretAccessKey\":\"***\",\"Version\":1}"] + (tokenize "echo '{\"AccessKeyId\":\"****\",\"SecretAccessKey\":\"***\",\"Version\":1}'")))) + (deftest process-test (testing "By default process returns string out and err, returning the exit code in a delay. Waiting for the process to end happens through realizing the @@ -117,6 +127,15 @@ (testing "output to string" (is (string? (-> (process ["ls"] {:out :string}) check + :out)))) + (testing "tokenize" + (is (string? (-> (process "ls -la" {:out :string}) + check + :out))) + (is (string? (-> ^{:out :string} ($ "ls -la" ) + check + :out))) + (is (string? (-> (sh "ls -la") :out))))) (defmacro ^:private jdk9+ []