upgrade!

project.clj

@@ -3,7 +3,7 @@
                 system implemented in Clojure. See http://pushlanguage.com"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
-  :dependencies [[org.clojure/clojure "1.9.0-alpha15"]
+  :dependencies [[org.clojure/clojure "1.8.0"]
                  [org.clojars.etosch/cosmos "1.0.0"]
                  [org.clojure/math.numeric-tower "0.0.4"]
                  [org.clojure/math.combinatorics "0.1.1"]
@@ -14,7 +14,7 @@
                  [clj-random "0.1.7"]
                  ;; https://mvnrepository.com/artifact/org.apache.commons/commons-math3
                  [org.apache.commons/commons-math3 "3.2"]
-                 [ici.recorder "0.2.0"]]
+                 [cheshire "5.7.1"]]
   :plugins [[lein-codox "0.9.1"]
             [lein-shell "0.5.0"]
             [lein-gorilla "0.4.0"]
@@ -38,7 +38,6 @@
                   ["change" "version" "leiningen.release/bump-version" "qualifier"]
                   ["shell" "git" "commit" "-am" "Version ${:version} [ci skip]"]
                   ["vcs" "push"]]
-  :aot [clojush.pushgp.record]
 ;;;;;;;;;; jvm settings for high performance, using most of the machine's RAM
 ;  :jvm-opts ~(let [mem-to-use
 ;                   (long (* (.getTotalPhysicalMemorySize

run-fly

@@ -25,7 +25,7 @@ homedir="/home/${fly_user}"
 rundir="$homedir/runs/$label-$number"
 repodir="$rundir/Clojush"
 outputdir="$rundir/output"
-parquet_uri="${CLOJUSH_PARQUET_URI}"
+parquet_uri="${CLOJUSH_PARQUET_URI:-}"
 ssh ${fly_user}@fly.hampshire.edu "mkdir -p $rundir"
 
 rsync  \

src/clojush/problems/software/replace_space_with_newline.clj

@@ -151,7 +151,9 @@
     (println ";;******************************")
     (printf ";; -*- Replace Space With Newline problem report - generation %s\n" generation)(flush)
     (println "Test total error for best:" best-total-test-error)
-    (println (format "Test mean error for best: %.5f" (double (/ best-total-test-error (count best-test-errors))))))
+    (println (format "Test mean error for best: %.5f"
+               (generation-data! [:best :mean-test-error]
+                 (double (/ best-total-test-error (count best-test-errors))))))
     (when (zero? (:total-error best))
       (doseq [[i error] (map vector
                              (range)

src/clojush/pushgp/pushgp.clj

@@ -149,7 +149,8 @@
       ;; set globals from parameters
       (reset-globals)
       (initial-report @push-argmap) ;; Print the inital report
-      (print-params (r/config-data! [:argmap] @push-argmap))
+      (r/uuid! (:run-uuid @push-argmap))
+      (print-params (r/config-data! [:argmap] (dissoc @push-argmap :run-uuid)))
       (check-genetic-operator-probabilities-add-to-one @push-argmap)
       (timer @push-argmap :initialization)
       (when (:print-timings @push-argmap)

src/clojush/pushgp/record.clj

@@ -9,108 +9,75 @@
 ;;; We build up the data we want to send for the generation/run over time
 ;;; and then send it off.
 (ns clojush.pushgp.record
-  (:require [ici-recorder]))
+  (:require [clojure.java.io]
+            [cheshire.core]
+            [cheshire.generate]))
+
+(cheshire.generate/add-encoder
+  clojure.lang.AFunction
+  cheshire.generate/encode-str)
+
+; https://github.com/clojure-cookbook/clojure-cookbook/blob/master/05_network-io/5-09_tcp-client.asciidoc
+(def writer
+  (-> (java.net.Socket. "35.185.102.205" 9990)
+    clojure.java.io/writer))
+;
+; (def generation-writer
+;   (-> (java.net.Socket. "localhost" 9992)
+;     clojure.java.io/writer))
+
+(defn- write-data! [writer data]
+  (cheshire.core/generate-stream data writer)
+  (.newLine writer)
+  (.flush writer))
 
 (def data (atom {}))
 
 
 ;; Stores a configuration options for the run, for the sequence of `ks` and value `v`
 (defn config-data! [ks v]
-  (swap! data assoc-in (cons :configuration ks) v)
+  (swap! data assoc-in (cons :config ks) v)
   v)
 
 ;; called at the begining of a new run.
 ;;
 ;; Resets the state and creates UUID
 (defn new-run! []
-  (reset! data {:uuid (str (java.util.UUID/randomUUID))}))
+  (reset! data {}))
+
+(defn uuid! [uuid]
+  (swap! data assoc :uuid uuid))
 
   ;; commented out until apache spark supports timestamp_millis
   ;; https://github.com/apache/spark/pull/15332
   ; (config-data! [:start-time] (java.time.Instant/now)))
 
-
-(def p-configuration
-  (array-map
-   :problem-file [true :string]
-   :argmap [true [:string false :string]]
-   :initialization-ms [false :long]
-   :registered-instructions [true [true :string]]
-   :versiom-numer [false :string]
-   :git-hash [false :string]))
-
-(def write-support-configuration (ici-recorder/->write-support p-configuration))
-
-
 ;; Records the run configuration with `ici-recorder`
-(defn end-config! []
-  (let [{:keys [configuration uuid]} @data]
-    (ici-recorder/record-run
-      write-support-configuration
-      uuid
-      configuration)))
+(defn end-config! [])
+;   (let [{:keys [configuration uuid]} @data]
+;     (write-data!
+;       config-writer
+;       (assoc configuration :uuid uuid))))
 
 ;; Called at the begining of a generation
 (defn new-generation! [index]
   (swap!
     data
-    (fn [m]
-      (assoc
-        (select-keys m [:uuid])
-        :index index))))
+    assoc
+    :index index
+    :generations {}))
 
 ;; stores some data about the generation
 (defn generation-data! [ks v]
   (swap! data assoc-in (cons :generation ks) v)
   v)
 
-(def p-error :double)
-(def p-errors [true p-error])
-(def p-plush-instruction-map
-  (array-map
-   :instruction [true :string]
-   :uuid [false :string]
-   :random-insertion [false :boolean]
-   :silent [false :boolean]
-   :random-closes [false :integer]
-   :parent-uuid [false :string]))
-(def p-genome [true p-plush-instruction-map])
-
-(def p-individual
-  (array-map
-   :genome [true p-genome]
-   :program [true :string]
-   :errors [false p-errors]
-   :total-error [false p-error]
-   :normalized-error [false p-error]
-   :meta-errors [false p-errors]
-   :history [false p-errors]
-   :ancestors [false [true p-genome]]
-   :uuid [false :string]
-   :parent-uuids [false [true :string]]
-   :genetic-operators [false :string]
-   :is-random-replacement [false :boolean]
-   :age [true :integer]))
-
-(def p-best
-  (array-map
-    :errors [true p-errors]
-    :test-errors [false p-errors]))
-
-(def p-generation
-  (array-map
-   :outcome [true :string]
-   :epsilons [false :double]
-   :population [true [true p-individual]]
-   :best [false p-best]))
-
-(def write-support-generation (ici-recorder/->write-support p-generation))
-
 ;; records the generation with `ici-recorder`
 (defn end-generation! []
-  (let [{:keys [generation uuid index]} @data]
-    (ici-recorder/record-generation
-      write-support-generation
-      uuid
-      index
-      generation)))
+  (let [{:keys [generation uuid index config]} @data]
+    (write-data!
+      writer
+      (assoc generation
+        :config-uuid uuid
+        :index index
+        :config config))))

src/clojush/pushgp/report.clj

@@ -354,6 +354,7 @@
           (lexicase-report population argmap))
     (when (= total-error-method :ifs) (implicit-fitness-sharing-report population argmap))
     (println (format "--- Best Program (%s) Statistics ---" (str "based on " (name err-fn))))
+    (r/generation-data! [:best :individual] best)
     (println "Best genome:" (print-genome best))
     (println "Best program:" (pr-str (not-lazy (:program best))))
     (when (> report-simplifications 0)
@@ -363,13 +364,14 @@
                                                           report-simplifications 
                                                           false 
                                                           1000))))))
-    (let [errors (r/generation-data! [:best :errors] (not-lazy (:errors best)))]
-      (when print-errors (println "Errors:" errors)))
+    (when print-errors (println "Errors:" (not-lazy (:errors best))))
     (when (and print-errors (not (empty? meta-error-categories)))
       (println "Meta-Errors:" (not-lazy (:meta-errors best))))
     (println "Total:" (:total-error best))
-    (println "Mean:" (float (/ (:total-error best)
-                               (count (:errors best)))))
+    (let [mean (r/generation-data! [:best :mean-error] (float (/ (:total-error best)
+                                                                 (count (:errors best)))))]
+
+      (println "Mean:"))
     (when (not= normalization :none)
       (println "Normalized error:" (:normalized-error best)))
     (case total-error-method
@@ -378,11 +380,12 @@
       :ifs (println "IFS-error:" (:weighted-error best))
       nil)
     (when print-history (println "History:" (not-lazy (:history best))))
-    (println "Genome size:" (count (:genome best)))
-    (println "Size:" (count-points (:program best)))
-    (printf "Percent parens: %.3f\n" 
-            (double (/ (count-parens (:program best)) 
-                       (count-points (:program best))))) ;Number of (open) parens / points
+    (println "Genome size:" (r/generation-data! [:best :genome-size] (count (:genome best))))
+    (println "Size:" (r/generation-data! [:best :program-size] (count-points (:program best))))
+    (printf "Percent parens: %.3f\n"
+            (r/generation-data! [:best :percent-parens]
+              (double (/ (count-parens (:program best))
+                         (count-points (:program best)))))) ;Number of (open) parens / points
     (println "--- Population Statistics ---")
     (when print-cosmos-data
       (println "Cosmos Data:" (let [quants (config/quantiles (count population))]