[MXNET-617] - Improve Clojure styles with cljfmt tools revise (apache…

…#11537) * Remove 'vi ci-test.sh' file * Add the scripts to reformat the style with cljfmt - Add lein-cljfmt-check to each project - Add lein-cljfmt-fix to each project * Add lein-cljfmt to the plugins vector * Add steps to keep style consistent to README.md * Run lein-cljfmt-fix on the main src/test codes - Run $MXROOT/contrib/clojure-package/lein-cljfmt-fix * Run lein cljfmt fix on the example projects - Run $MXNET_ROOT/contrib/clojure-package/examples/lein-cljfmt-fix * Use only one script in the base directory. - Thanks to @marcoabreu for the suggestion/review * Minor update to kick off the new build
XinYao1994 · Jul 3, 2018 · 43e3703 · 43e3703
1 parent dc8ada3
commit 43e3703
Show file tree

Hide file tree

Showing 62 changed files with 630 additions and 744 deletions.
diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md
@@ -168,6 +168,16 @@ _Note: There is an error thrown in the generated code due to some loading issues
 
 To run the Code Coverage tool. Run `lein cloverage`.
 
+## Tools to keep style consistent
+
+To keep the style consistent for the project we include the script that make it easier.
+There are two script in the base of the project and in each examples.
+
+To run it just see the following file. `lein-cljfmt-check` and `lein-cljfmt-fix`.
+The first command will run and check and confirm if the code needed to be updated to reflect the community style guide.
+The second command will apply the change and fix any inconsistent indentation in place. This is recommendd to be done
+before the submit a new pull request so we can keep the style consistent throughout the project.
+
 ## FAQ
 
 

diff --git a/contrib/clojure-package/examples/cnn-text-classification/project.clj b/contrib/clojure-package/examples/cnn-text-classification/project.clj
@@ -17,7 +17,8 @@
 
 (defproject cnn-text-classification "0.1.0-SNAPSHOT"
   :description "CNN text classification with MXNet"
+  :plugins [[lein-cljfmt "0.5.7"]]
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.0-SNAPSHOT"]]
-  :main cnn-text-classification.classifier
-  :pedantic? :skip)
+  :pedantic? :skip
+  :main cnn-text-classification.classifier)
diff --git a/...ojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj b/...ojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj
@@ -85,7 +85,7 @@
                                         :data-batch-size batch-size
                                         :last-batch-handle "pad"})
         test-data (mx-io/ndarray-iter [(get-in shuffled [:test :data])]
-                                      {:label[(get-in  shuffled [:test :label])]
+                                      {:label [(get-in  shuffled [:test :label])]
                                        :label-name "softmax_label"
                                        :data-batch-size batch-size
                                        :last-batch-handle "pad"})]
@@ -107,8 +107,6 @@
     ;; runs all the examples
     #_(train-convnet {:embedding-size 50 :batch-size 100 :test-size 1000 :num-epoch 10})))
 
-
 (comment
-  (train-convnet {:devs [(context/cpu)] :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000})
-  )
+  (train-convnet {:devs [(context/cpu)] :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000}))
 
diff --git a/...jure-package/examples/cnn-text-classification/src/cnn_text_classification/data_helper.clj b/...jure-package/examples/cnn-text-classification/src/cnn_text_classification/data_helper.clj
@@ -42,7 +42,6 @@
         (.append sb (new String bs 0 i))))
     (.toString sb)))
 
-
 (defn get-float [b]
   (-> 0
       (bit-or (bit-shift-left (bit-and (aget b 0) 0xff) 0))
@@ -70,31 +69,27 @@
       (println "Finished")
       {:num-embed dim :word2vec word2vec})))
 
-
 (defn clean-str [s]
   (-> s
-    (string/replace #"^A-Za-z0-9(),!?'`]" " ")
-    (string/replace #"'s" " 's")
-    (string/replace #"'ve" " 've")
-    (string/replace #"n't" " n't")
-    (string/replace #"'re" " 're")
-    (string/replace #"'d" " 'd")
-    (string/replace #"'ll" " 'll")
-    (string/replace #"," " , ")
-    (string/replace #"!" " ! ")
-    (string/replace #"\(" " ( ")
-    (string/replace #"\)" " ) ")
-    (string/replace #"\?" " ? ")
-    (string/replace #" {2,}" " ")
-    (string/trim)))
-
-
- ;; Loads MR polarity data from files, splits the data into words and generates labels.
+      (string/replace #"^A-Za-z0-9(),!?'`]" " ")
+      (string/replace #"'s" " 's")
+      (string/replace #"'ve" " 've")
+      (string/replace #"n't" " n't")
+      (string/replace #"'re" " 're")
+      (string/replace #"'d" " 'd")
+      (string/replace #"'ll" " 'll")
+      (string/replace #"," " , ")
+      (string/replace #"!" " ! ")
+      (string/replace #"\(" " ( ")
+      (string/replace #"\)" " ) ")
+      (string/replace #"\?" " ? ")
+      (string/replace #" {2,}" " ")
+      (string/trim)));; Loads MR polarity data from files, splits the data into words and generates labels.
  ;; Returns split sentences and labels.
 (defn load-mr-data-and-labels [path max-examples]
   (println "Loading all the movie reviews from " path)
   (let [positive-examples (mapv #(string/trim %) (-> (slurp (str path "/rt-polarity.pos"))
-                                                    (string/split #"\n")))
+                                                     (string/split #"\n")))
         negative-examples (mapv #(string/trim %) (-> (slurp (str path "/rt-polarity.neg"))
                                                      (string/split #"\n")))
         positive-examples (into [] (if max-examples (take max-examples positive-examples) positive-examples))
@@ -118,10 +113,7 @@
                     (if (pos? diff)
                       (into s (repeat diff padding-word))
                       s)))
-          sentences)))
-
-
- ;; Map sentences and labels to vectors based on a pretrained embeddings
+          sentences)));; Map sentences and labels to vectors based on a pretrained embeddings
 (defn build-input-data-with-embeddings [sentences embedding-size embeddings]
   (mapv (fn [sent]
           (mapv (fn [word] (or (get embeddings word)

diff --git a/contrib/clojure-package/examples/gan/project.clj b/contrib/clojure-package/examples/gan/project.clj
@@ -17,6 +17,7 @@
 
 (defproject gan "0.1.0-SNAPSHOT"
   :description "GAN MNIST with MXNet"
+  :plugins [[lein-cljfmt "0.5.7"]]
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.0-SNAPSHOT"]
                  [nu.pattern/opencv "2.4.9-7"]]

diff --git a/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj b/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj
@@ -32,7 +32,6 @@
             [org.apache.clojure-mxnet.context :as context])
   (:gen-class))
 
-
 ;; based off of https://medium.com/@julsimon/generative-adversarial-networks-on-apache-mxnet-part-1-b6d39e6b5df1
 
 
@@ -57,7 +56,7 @@
 (comment
 
   ;;This is for figuring out the convolution and deconvolution layers to convert the image sizes
-  
+
   (defn conv-output-size [input-size kernel-size padding stride]
     (float (inc (/ (- (+ input-size (* 2 padding)) kernel-size) stride))))
 
@@ -86,7 +85,6 @@
 (def lr  0.0005) ;; learning rate
 (def beta1 0.5)
 
-
 (def label (sym/variable "label"))
 
 (defn discriminator []
@@ -130,26 +128,24 @@
       label [(ndarray/ones [batch-size 100 1 1])]]
   (def my-iter (mx-io/ndarray-iter data)))
 
-
 (defn save-img-gout [i n x]
   (do
     (viz/im-sav {:title (str "gout-" i "-" n)
-             :output-path output-path
-             :x x
-             :flip false})))
+                 :output-path output-path
+                 :x x
+                 :flip false})))
 
 (defn save-img-diff [i n x]
   (do (viz/im-sav {:title (str "diff-" i "-" n)
-            :output-path output-path
-               :x x
-               :flip false})))
+                   :output-path output-path
+                   :x x
+                   :flip false})))
 
 (defn save-img-data [i n batch]
   (do (viz/im-sav {:title (str "data-" i "-" n)
-               :output-path output-path
-               :x (first (mx-io/batch-data batch))
-               :flip false})))
-
+                   :output-path output-path
+                   :x (first (mx-io/batch-data batch))
+                   :flip false})))
 
 (defn calc-diff [i n diff-d]
   (let [diff (ndarray/copy diff-d)
@@ -161,8 +157,6 @@
 
       (save-img-diff i n calc-diff))))
 
-
-
 (defn train [devs]
   (let [mod-d  (-> (m/module (discriminator) {:contexts devs :data-names ["data"] :label-names ["label"]})
                    (m/bind {:data-shapes (mx-io/provide-data mnist-iter)
@@ -171,55 +165,53 @@
                    (m/init-params {:initializer (init/normal 0.02)})
                    (m/init-optimizer {:optimizer (opt/adam {:learning-rate lr :wd 0.0 :beta1 beta1})}))
         mod-g (-> (m/module (generator) {:contexts devs :data-names ["rand"] :label-names nil})
-                   (m/bind {:data-shapes (mx-io/provide-data rand-noise-iter)})
-                   (m/init-params {:initializer (init/normal 0.02)})
-                   (m/init-optimizer {:optimizer (opt/adam {:learning-rate lr :wd 0.0 :beta1 beta1})}))]
+                  (m/bind {:data-shapes (mx-io/provide-data rand-noise-iter)})
+                  (m/init-params {:initializer (init/normal 0.02)})
+                  (m/init-optimizer {:optimizer (opt/adam {:learning-rate lr :wd 0.0 :beta1 beta1})}))]
 
     (println "Training for " num-epoch " epochs...")
     (doseq [i (range num-epoch)]
-     (mx-io/reduce-batches mnist-iter
-                           (fn [n batch]
-                             (let [rbatch (mx-io/next rand-noise-iter)
-                                   out-g (-> mod-g
-                                             (m/forward rbatch)
-                                             (m/outputs))
+      (mx-io/reduce-batches mnist-iter
+                            (fn [n batch]
+                              (let [rbatch (mx-io/next rand-noise-iter)
+                                    out-g (-> mod-g
+                                              (m/forward rbatch)
+                                              (m/outputs))
                                    ;; update the discriminiator on the fake
-                                   grads-f  (mapv #(ndarray/copy (first %)) (-> mod-d
-                                                                                (m/forward {:data (first out-g) :label [(ndarray/zeros [batch-size])]})
-                                                                                (m/backward)
-                                                                                (m/grad-arrays)))
+                                    grads-f  (mapv #(ndarray/copy (first %)) (-> mod-d
+                                                                                 (m/forward {:data (first out-g) :label [(ndarray/zeros [batch-size])]})
+                                                                                 (m/backward)
+                                                                                 (m/grad-arrays)))
                                    ;; update the discrimintator on the real
-                                   grads-r (-> mod-d
-                                               (m/forward {:data (mx-io/batch-data batch) :label [(ndarray/ones [batch-size])]})
-                                               (m/backward)
-                                               (m/grad-arrays))
-                                   _ (mapv (fn [real fake] (let [r (first real)]
-                                                             (ndarray/set r (ndarray/+ r fake)))) grads-r grads-f)
-                                   _ (m/update mod-d)
+                                    grads-r (-> mod-d
+                                                (m/forward {:data (mx-io/batch-data batch) :label [(ndarray/ones [batch-size])]})
+                                                (m/backward)
+                                                (m/grad-arrays))
+                                    _ (mapv (fn [real fake] (let [r (first real)]
+                                                              (ndarray/set r (ndarray/+ r fake)))) grads-r grads-f)
+                                    _ (m/update mod-d)
                                    ;; update the generator
-                                   diff-d (-> mod-d
-                                              (m/forward {:data (first out-g) :label [(ndarray/ones [batch-size])]})
-                                              (m/backward)
-                                              (m/input-grads))
-                                   _ (-> mod-g
-                                         (m/backward (first diff-d))
-                                         (m/update))]
-                               (when (zero? (mod n 100))
-                                 (println "iteration = " i  "number = " n)
-                                 (save-img-gout i n (ndarray/copy (ffirst out-g)))
-                                 (save-img-data i n batch)
-                                 (calc-diff i n (ffirst diff-d)))
-                               (inc n)))))))
+                                    diff-d (-> mod-d
+                                               (m/forward {:data (first out-g) :label [(ndarray/ones [batch-size])]})
+                                               (m/backward)
+                                               (m/input-grads))
+                                    _ (-> mod-g
+                                          (m/backward (first diff-d))
+                                          (m/update))]
+                                (when (zero? (mod n 100))
+                                  (println "iteration = " i  "number = " n)
+                                  (save-img-gout i n (ndarray/copy (ffirst out-g)))
+                                  (save-img-data i n batch)
+                                  (calc-diff i n (ffirst diff-d)))
+                                (inc n)))))))
 
 (defn -main [& args]
   (let [[dev dev-num] args
         devs (if (= dev ":gpu")
                (mapv #(context/gpu %) (range (Integer/parseInt (or dev-num "1"))))
                (mapv #(context/cpu %) (range (Integer/parseInt (or dev-num "1")))))]
-     (println "Running with context devices of" devs)
+    (println "Running with context devices of" devs)
     (train devs)))
 
 (comment
-  (train [(context/cpu)])
-
-  )
+  (train [(context/cpu)]))
diff --git a/contrib/clojure-package/examples/gan/src/gan/viz.clj b/contrib/clojure-package/examples/gan/src/gan/viz.clj
@@ -27,7 +27,6 @@
 ;;; Viz stuff
 (OpenCV/loadShared)
 
-
 (defn clip [x]
   (->> x
        (mapv #(* 255 %))
@@ -69,7 +68,7 @@
         [n c h w] shape
         totals (* h w)
         raw-data (byte-array (clip (ndarray/to-array x)))
-        row (.intValue(Math/sqrt n))
+        row (.intValue (Math/sqrt n))
         col row
         line-arrs (into [] (partition (* col c totals) raw-data))
         line-mats (mapv (fn [line]

diff --git a/contrib/clojure-package/examples/imclassification/project.clj b/contrib/clojure-package/examples/imclassification/project.clj
@@ -17,7 +17,8 @@
 
 (defproject imclassification "0.1.0-SNAPSHOT"
   :description "Clojure examples for image classification"
+  :plugins [[lein-cljfmt "0.5.7"]]
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.0-SNAPSHOT"]]
-  :main imclassification.train-mnist
-  :pedantic? :skip)
+  :pedantic? :skip
+  :main imclassification.train-mnist)
diff --git a/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj b/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj
@@ -111,5 +111,4 @@
     (start devs)))
 
 (comment
-  (start [(context/cpu)])
-  )
+  (start [(context/cpu)]))
diff --git a/contrib/clojure-package/examples/module/project.clj b/contrib/clojure-package/examples/module/project.clj
@@ -17,8 +17,9 @@
 
 (defproject module-examples "0.1.0-SNAPSHOT"
   :description "Clojure examples for module"
+  :plugins [[lein-cljfmt "0.5.7"]]
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.0-SNAPSHOT"]]
-  :main  mnist-mlp
-  :pedantic? :skip)
+  :pedantic? :skip
+  :main mnist-mlp)