Merge pull request #311 from Deraen/gpg-binary

Use gpg binary for signing and reading credentials from encrypted file

CHANGES.md

@@ -1,5 +1,17 @@
 # Changes
 
+## Unreleased
+
+- Added support for reading repository credentials from encrypted file
+(`~/.boot/credentials.clj.gpg`) and environment variables.
+- **BREAKING**: `gpg` binary is now used for signing jars and reading encrypted
+credentials file
+    - Deprecated `push` option `gpg-keyring`
+- `push` task can now be provided with `repo-map` option to set the deployment
+repository. This is useful for example in case a repository needs different
+settings for downloading dependencies and deploying, like additional
+credentials.
+
 ## 2.4.2
 
 - Fix issue where the wrong classloader was being used to

boot/aether/project.clj

@@ -14,5 +14,6 @@
   :license      {:name "Eclipse Public License"
                  :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure      "1.6.0"  :scope "compile"]
+                 [boot/base                ~version :scope "provided"]
                  [boot/pod                 ~version :scope "compile"]
                  [com.cemerick/pomegranate "0.3.0"  :scope "compile"]])

boot/aether/src/boot/aether.clj

@@ -6,11 +6,14 @@
    [cemerick.pomegranate.aether :as aether]
    [boot.util                   :as util]
    [boot.pod                    :as pod]
+   [boot.gpg                    :as gpg]
    [boot.from.io.aviso.ansi     :as ansi]
    [boot.kahnsort               :as ksort])
   (:import
+   [boot App]
    [java.io File]
    [java.util.jar JarFile]
+   [java.util.regex Pattern]
    [org.sonatype.aether.resolution DependencyResolutionException]))
 
 (def offline?             (atom false))
@@ -59,13 +62,75 @@
           :password        password
           :non-proxy-hosts (get-non-proxy-hosts)}))))
 
+(defn ^{:boot/from :technomancy/leiningen} credentials-fn
+  "Decrypt map from credentials.clj.gpg in Boot home if present."
+  ([] (let [cred-file (io/file (App/bootdir) "credentials.clj.gpg")]
+        (if (.exists cred-file)
+          (credentials-fn cred-file))))
+  ([file]
+   (let [{:keys [out err exit]} (gpg/gpg "--quiet" "--batch"
+                                         "--decrypt" "--" (str file))]
+     (if (pos? exit)
+       (binding [*out* *err*]
+         (util/warn (format (str "Could not decrypt credentials from %s\n"
+                                 "%s\n"
+                                 "See `boot gpg --help` for how to install gpg.")
+                            (str file) err)))
+       (read-string out)))))
+
+(def credentials (memoize credentials-fn))
+
+(defn- ^{:boot/from :technomancy/leiningen} match-credentials [settings auth-map]
+  (get auth-map (:url settings)
+       (first (for [[re? cred] auth-map
+                    :when (and (instance? Pattern re?)
+                               (re-find re? (:url settings)))]
+                cred))))
+
+(defn- ^{:boot/from :technomancy/leiningen} resolve-credential
+  "Resolve key-value pair from result into a credential, updating result."
+  [source-settings result [k v]]
+  (letfn [(resolve [v]
+            (cond (= :env v)
+                  (System/getenv (str "BOOT_" (string/upper-case (name k))))
+
+                  (and (keyword? v) (= "env" (namespace v)))
+                  (System/getenv (string/upper-case (name v)))
+
+                  (= :gpg v)
+                  (get (match-credentials source-settings (credentials)) k)
+
+                  (coll? v) ;; collection of places to look
+                  (->> (map resolve v)
+                       (remove nil?)
+                       first)
+
+                  :else v))]
+    (if (#{:username :password :passphrase :private-key-file} k)
+      (assoc result k (resolve v))
+      (assoc result k v))))
+
+(defn ^{:boot/from :technomancy/leiningen} resolve-credentials
+  "Applies credentials from the environment or ~/.boot/credentials.clj.gpg
+  as they are specified and available."
+  [settings]
+  (let [gpg-creds (if (= :gpg (:creds settings))
+                    (match-credentials settings (credentials)))
+        resolved (reduce (partial resolve-credential settings)
+                         (empty settings)
+                         settings)]
+    (if gpg-creds
+      (dissoc (merge gpg-creds resolved) :creds)
+      resolved)))
+
 (defn resolve-dependencies*
   [env]
   (try
     (aether/resolve-dependencies
       :coordinates       (:dependencies env)
       :repositories      (->> (or (:repositories env) @default-repositories)
                            (map (juxt first (fn [[x y]] (if (map? y) y {:url y}))))
+                           (map (juxt first (fn [[x y]] (resolve-credentials y))))
                            (map (juxt first (fn [[x y]] (update-in y [:update] #(or % @update?))))))
       :local-repo        (or (:local-repo env) @local-repo nil)
       :offline?          (or @offline? (:offline? env))
@@ -167,17 +232,18 @@
       :local-repo  (or (:local-repo env) @local-repo nil))))
 
 (defn deploy
-  [env repo jarfile & [artifact-map]]
+  [env [repo-id repo-settings] jarfile & [artifact-map]]
   (let [{:keys [project version]}
         (-> jarfile pod/pom-properties pod/pom-properties-map)
         pomfile (doto (File/createTempFile "pom" ".xml")
-                  .deleteOnExit (spit (pod/pom-xml jarfile)))]
+                  .deleteOnExit (spit (pod/pom-xml jarfile)))
+        repo-settings (resolve-credentials repo-settings)]
     (aether/deploy
       :coordinates  [project version]
       :jar-file     (io/file jarfile)
       :pom-file     (io/file pomfile)
       :artifact-map artifact-map
-      :repository   [repo]
+      :repository   {repo-id repo-settings}
       :local-repo   (or (:local-repo env) @local-repo nil))))
 
 (def ^:private wagon-files (atom #{}))

boot/core/src/boot/cli.clj

@@ -77,17 +77,25 @@
         (symbol? type) (assoc m k (parse-type type v))
         (coll? type)   (update-in m [k] (fnil conj (empty type)) (parse-type (first type) v))))))
 
-(defn- format-doc [optarg type doc]
+(defn- deprecated [short]
+  (:deprecated (meta short)))
+
+(defn- deprecated-doc [doc]
+  (str "DEPRECATED: " doc))
+
+(defn- format-doc [short optarg type doc]
   (let [atom? (symbol? type)
         flag? (not optarg)
-        incr? (and flag? (= 'int type))]
-    (cond
-      incr? (format "Increase %s" (decap doc))
-      flag? doc
-      atom? (format "Set %s to %s." (depunc (decap doc)) optarg)
-      :else (let [f "Conj %s onto %s"
-                  v ((parse-fn optarg) (str optarg))]
-              (format f (if (string? v) v (pr-str (mapv symbol v))) (decap doc))))))
+        incr? (and flag? (= 'int type))
+        docstring (cond
+                    incr? (format "Increase %s" (decap doc))
+                    flag? doc
+                    atom? (format "Set %s to %s." (depunc (decap doc)) optarg)
+                    :else (let [f "Conj %s onto %s"
+                                v ((parse-fn optarg) (str optarg))]
+                            (format f (if (string? v) v (pr-str (mapv symbol v))) (decap doc))))]
+    (cond-> docstring
+      (deprecated short) deprecated-doc)))
 
 (defn- argspec->cli-argspec
   ([short long type doc]
@@ -99,7 +107,7 @@
       [:id           (keyword long)
        :long-opt     (str "--" long)
        :required     (when optarg (str optarg))
-       :desc         (format-doc optarg type doc)
+       :desc         (format-doc short optarg type doc)
        :parse-fn     `(#'parse-fn ~(when optarg (list 'quote optarg)))
        :assoc-fn     `(#'assoc-fn ~(when optarg (list 'quote optarg)) '~type)]))))
 
@@ -113,11 +121,19 @@
           (throw (IllegalArgumentException.
                    ~(format "option :%s must be of type %s" long type)))))))
 
+(defn- argspec->deprecation-warning
+  ([short long type doc]
+   (argspec->deprecation-warning short long nil type doc))
+  ([short long optarg type doc]
+   (if-let [deprecated (deprecated short)]
+     `(when-not (nil? ~long)
+        (util/warn ~(format "option %s is deprecated. %s\n" long (if (string? deprecated) deprecated "")))))))
+
 (defn- argspec->summary
   ([short long type doc]
      (argspec->summary short long nil type doc))
   ([short long optarg type doc]
-     [(str ":" long) (str (when (:! (meta type)) "^:! ")) (str type) doc]))
+     [(str ":" long) (str (when (:! (meta type)) "^:! ")) (str type) (cond-> doc (deprecated short) deprecated-doc)]))
 
 (defn- argspec-seq [args]
   (when (seq args)
@@ -185,6 +201,7 @@
                    ~'*args*  (:arguments parsed#)
                    ~'*usage* #(print ~cli-doc)]
                ~@(mapv (partial apply argspec->assert) argspecs)
+               ~@(mapv (partial apply argspec->deprecation-warning) argspecs)
                (if-not ~'help (do ~@body) (~'*usage*))))
            (with-meta ~varmeta)))))
 

boot/core/src/boot/task/built_in.clj

@@ -13,6 +13,7 @@
    [boot.util            :as util]
    [boot.from.table.core :as table]
    [boot.task-helpers    :as helpers]
+   [boot.gpg             :as gpg]
    [boot.pedantic        :as pedantic])
   (:import
    [java.io File]
@@ -684,17 +685,37 @@
 (core/deftask push
   "Deploy jar file to a Maven repository.
 
-  The repo option is required. If the file option is not specified the task will
+  If the file option is not specified the task will
   look for jar files created by the build pipeline. The jar file(s) must contain
-  pom.xml entries."
+  pom.xml entries.
+
+  The repo option is required. The repo option is used to get repository map
+  from Boot envinronment. Additional repo-map option can be used to add
+  options, like credentials, or to provide complete repo-map if Boot
+  envinronment doesn't hold the named repository.
+
+  Repository map special options:
+
+  - `:creds :gpg`
+    - Username and password are read from encrypted file at ~/.boot/credentials.clj.gpg.
+  - `:username :gpg :password :gpg`
+    - Username and password are read from encrypted file at ~/.boot/credentials.clj.gpg.
+  - `:username :env :password :env`
+    - Username is read from envinronment variable `BOOT_{{REPO_NAME}}_USERNAME` and password from `BOOT_{{REPO_NAME}}_PASSWORD`.
+  - `:username :env/foo :password :env/bar`
+    - Username is read from envinronment variable `FOO` and password from `BAR`.
+  - `:username [:gpg :env :env/foo]`
+    - Username is read from first available source."
 
   [f file PATH            str      "The jar file to deploy."
    F file-regex MATCH     #{regex} "The set of regexes of paths to deploy."
    g gpg-sign             bool     "Sign jar using GPG private key."
-   k gpg-user-id NAME     str      "The name used to find the GPG key."
+   k gpg-user-id KEY      str      "The name or key-id used to select the signing key."
+   ^{:deprecated "Check GPG help about changing GNUPGHOME."}
    K gpg-keyring PATH     str      "The path to secring.gpg file to use for signing."
    p gpg-passphrase PASS  str      "The passphrase to unlock GPG signing key."
-   r repo ALIAS           str      "The alias of the deploy repository."
+   r repo NAME            str      "The name of the deploy repository."
+   e repo-map REPO        edn      "The repository map of the deploy repository."
    t tag                  bool     "Create git tag for this version."
    B ensure-branch BRANCH str      "The required current git branch."
    C ensure-clean         bool     "Ensure that the project git repo is clean."
@@ -712,9 +733,13 @@
                                 (core/by-ext [".jar"])
                                 ((if (seq file-regex) #(core/by-re file-regex %) identity))
                                 (map core/tmp-file)))
-              repo-map (->> (core/get-env :repositories) (into {}))
-              r        (get repo-map repo)]
-          (when-not (and r (seq jarfiles))
+              ; Get options from Boot env by repo name
+              r        (get (->> (core/get-env :repositories) (into {})) repo)
+              repo-map (merge
+                         ; Repo option in env might be given as only url instead of map
+                         (if (map? r) r {:url r})
+                         repo-map)]
+          (when-not (and repo-map (seq jarfiles))
             (throw (Exception. "missing jar file or repo not found")))
           (doseq [f jarfiles]
             (let [{{t :tag} :scm
@@ -726,7 +751,8 @@
                   snapshot?    (.endsWith v "-SNAPSHOT")
                   artifact-map (when gpg-sign
                                  (util/info "Signing %s...\n" (.getName f))
-                                 (helpers/sign-jar tgt f gpg-passphrase gpg-keyring gpg-user-id))]
+                                 (gpg/sign-jar tgt f {:gpg-key gpg-user-id
+                                                      :gpg-passphrase gpg-passphrase}))]
               (assert (or (not ensure-branch) (= b ensure-branch))
                       (format "current git branch is %s but must be %s" b ensure-branch))
               (assert (or (not ensure-clean) clean?)
@@ -743,7 +769,7 @@
                       (format "jar version doesn't match project version (%s, %s)" v ensure-version))
               (util/info "Deploying %s...\n" (.getName f))
               (pod/with-call-worker
-                (boot.aether/deploy ~(core/get-env) ~[repo r] ~(.getPath f) ~artifact-map))
+                (boot.aether/deploy ~(core/get-env) ~[repo repo-map] ~(.getPath f) ~artifact-map))
               (when tag
                 (if (and tags (= commit (get tags tag)))
                   (util/info "Tag %s already created for %s\n" tag commit)

boot/core/src/boot/task_helpers.clj

@@ -41,18 +41,6 @@
   [prompt]
   (String/valueOf (.readPassword (System/console) prompt nil)))
 
-(defn sign-jar [out jar pass keyring user-id]
-  (let [prompt (pod/with-call-worker
-                 (boot.pgp/prompt-for ~keyring ~user-id))
-        pass   (or pass (read-pass prompt))]
-    (pod/with-call-worker
-      (boot.pgp/sign-jar
-        ~(.getPath out)
-        ~(.getPath jar)
-        ~pass
-        :keyring ~keyring
-        :user-id ~user-id))))
-
 (defn print-fileset
   [fileset]
   (letfn [(tree [xs]

boot/pod/src/boot/gpg.clj

@@ -0,0 +1,66 @@
+(ns boot.gpg
+  (:require
+   [clojure.java.io   :as io]
+   [clojure.java.shell :as shell]
+   [boot.pod          :as pod]
+   [boot.util         :as util])
+  (:import [java.io StringReader File]))
+
+(defn ^{:boot/from :technomancy/leiningen} gpg-program
+  "Lookup the gpg program to use, defaulting to 'gpg'"
+  []
+  (or (System/getenv "BOOT_GPG") "gpg"))
+
+(defn- ^{:boot/from :technomancy/leiningen} get-english-env []
+  "Returns environment variables as a map with clojure keywords and LANGUAGE set to 'en'"
+  (let [env (System/getenv)
+        keywords (map #(keyword %) (keys env))]
+    (merge (zipmap keywords (vals env))
+           {:LANGUAGE "en"})))
+
+(defn ^{:boot/from :technomancy/leiningen} gpg
+  "Shells out to (gpg-program) with the given arguments"
+  [& args]
+  (let [env (get-english-env)]
+    (try
+      (shell/with-sh-env env
+        (apply shell/sh (gpg-program) args))
+      (catch java.io.IOException e
+        {:exit 1 :err (.getMessage e)}))))
+
+(defn ^{:boot/from :technomancy/leiningen} signing-args
+  "Produce GPG arguments for signing a file."
+  [file opts]
+  (let [key-spec (if-let [key (:gpg-key opts)]
+                   ["--default-key" key])
+        passphrase-spec (if-let [pass (:gpg-passphrase opts)]
+                          ["--passphrase-fd" "0"])
+        passphrase-in-spec (if-let [pass (:gpg-passphrase opts)]
+                             [:in (StringReader. pass)])]
+    `["--yes" "-ab" ~@key-spec ~@passphrase-spec "--" ~file ~@passphrase-in-spec]))
+
+(defn ^{:boot/from :technomancy/leiningen} sign
+  "Create a detached signature and return the signature file name."
+  [file opts]
+  (let [{:keys [err exit]} (apply gpg (signing-args file opts))]
+    (when-not (zero? exit)
+      (util/fail (str "Could not sign " file "\n" err
+                      "\n\nIf you don't expect people to need to verify the "
+                      "authorship of your jar, don't set :gpg-sign option of push task to true.\n")))
+    (str file ".asc")))
+
+(defn sign-jar
+  [outdir jarfile opts]
+  (shell/with-sh-dir
+    outdir
+    (let [jarname (.getName jarfile)
+          jarout  (io/file outdir (str jarname ".asc"))
+          pomfile (doto (File/createTempFile "pom" ".xml")
+                    (.deleteOnExit)
+                    (spit (pod/pom-xml jarfile)))
+          pomout  (io/file outdir (.replaceAll jarname "\\.jar$" ".pom.asc"))
+          sign-it #(sign (.getPath %) opts)]
+      (spit pomout (sign-it pomfile))
+      (spit jarout (sign-it jarfile))
+      {[:extension "jar.asc"] (.getPath jarout)
+       [:extension "pom.asc"] (.getPath pomout)})))

boot/worker/project.clj

@@ -25,7 +25,6 @@
                  [clj-jgit                    "0.8.0"]
                  [clj-yaml                    "0.4.0"]
                  [javazoom/jlayer             "1.0.1"]
-                 [mvxcvi/clj-pgp              "0.5.4"]
                  [net.java.dev.jna/jna        "4.1.0"]
                  [alandipert/desiderata       "1.0.2"]
                  [org.clojure/data.xml        "0.0.8"]