Merge 8f8d92c into 505abf3

.gitignore

@@ -24,4 +24,6 @@ cov/*
 out
 .idea
 clojush.iml
-.gorilla-port
+.gorilla-port
+/clojush/
+.ipynb_checkpoints/

project.clj

@@ -1,6 +1,6 @@
-(defproject clojush "2.30.0-1-SNAPSHOT" 
+(defproject clojush "2.30.0-1-SNAPSHOT"
   :description "The Push programming language and the PushGP genetic programming
-                system implemented in Clojure. See http://pushlanguage.com" 
+                system implemented in Clojure. See http://pushlanguage.com"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.8.0"]
@@ -12,8 +12,9 @@
                  [clojure-csv "2.0.1"]
                  [org.clojure/data.json "0.2.6"]
                  [clj-random "0.1.7"]
-                 ;; https://mvnrepository.com/artifact/org.apache.commons/commons-math3 
-                 [org.apache.commons/commons-math3 "3.2"]]
+                 ;; https://mvnrepository.com/artifact/org.apache.commons/commons-math3
+                 [org.apache.commons/commons-math3 "3.2"]
+                 [cheshire "5.7.1"]]
   :plugins [[lein-codox "0.9.1"]
             [lein-shell "0.5.0"]
             [lein-gorilla "0.4.0"]
@@ -53,4 +54,3 @@
   ;;"-XX:+UseG1GC"
   ;:jvm-opts ["-Xmx12g" "-Xms12g" "-XX:+UseParallelGC"]
   :main clojush.core)
-

run-fly

@@ -25,7 +25,7 @@ homedir="/home/${fly_user}"
 rundir="$homedir/runs/$label-$number"
 repodir="$rundir/Clojush"
 outputdir="$rundir/output"
-
+parquet_uri="${CLOJUSH_PARQUET_URI:-}"
 ssh ${fly_user}@fly.hampshire.edu "mkdir -p $rundir"
 
 rsync  \
@@ -49,7 +49,7 @@ ssh ${fly_user}@fly.hampshire.edu /opt/pixar/tractor-blade-1.7.2/tractor-spool.p
   --jobcwd="${repodir}" \
   --priority=1 \
   --range 1-${n} \
-  -c "bash -c 'env JAVA_CMD=/usr/java/latest/bin/java /share/apps/bin/lein run $lein_command >  $outputdir/RANGE.out 2> $outputdir/RANGE.err'"
+  -c "bash -c 'env JAVA_CMD=/usr/java/latest/bin/java CLOJUSH_PARQUET_URI=$parquet_uri /share/apps/bin/lein run $lein_command :label $label >  $outputdir/RANGE.out 2> $outputdir/RANGE.err'"
 
 
 echo "Job ID: ${number}"

src/clojush/args.clj

@@ -406,7 +406,7 @@
           ;; When true, will exit the run when there is an individual with a zero-error vector
 
           ;;----------------------------------------
-          ;; Arguments related to printing JSON, EDN, or CSV logs
+          ;; Arguments related to printing JSON, EDN, CSV, or Parquet logs
           ;;----------------------------------------
 
           :print-csv-logs false
@@ -445,6 +445,9 @@
 
           :json-log-program-strings false
           ;; If true, JSON logs will include program strings for each individual.
+
+          :label nil
+          ;; If set, parquet will save this label as a string as metadata about the run
           )))
 
 (defn load-push-argmap

src/clojush/core.clj

@@ -16,6 +16,7 @@
 ;; for more details.
 
 (ns clojush.core
+  (:require [clojush.pushgp.record :as r])
   (:use [clojush.pushgp pushgp report])
   (:gen-class))
 
@@ -30,12 +31,13 @@
    This allows one to run an example with a call from the OS shell prompt like:
        lein run examples.simple-regression :population-size 3000"
   [& args]
+  (r/new-run!)
   (println "Command line args:" (apply str (interpose \space args)))
   (let [param-list (map #(if (.endsWith % ".ser")
                            (str %)
                            (read-string %))
                         (rest args))]
-    (require (symbol (first args)))
+    (require (symbol (r/config-data! [:problem-file] (first args))))
     (let [example-params (eval (symbol (str (first args) "/argmap")))
           params (merge example-params (apply sorted-map param-list))]
       (println "######################################")

src/clojush/problems/software/replace_space_with_newline.clj

@@ -15,7 +15,8 @@
         [clojush pushstate interpreter random util globals]
         clojush.instructions.tag
         clojure.math.numeric-tower)
-    (:require [clojure.string :as string]))
+    (:require [clojure.string :as string]
+              [clojush.pushgp.record :refer [generation-data!]]))
 
 ;; Define test cases
 (defn replace-space-input
@@ -145,12 +146,14 @@
   "Custom generational report."
   [best population generation error-function report-simplifications]
   (let [best-program (not-lazy (:program best))
-        best-test-errors (error-function best-program :test)
+        best-test-errors (generation-data! [:best :test-errors] (error-function best-program :test))
         best-total-test-error (apply +' best-test-errors)]
     (println ";;******************************")
     (printf ";; -*- Replace Space With Newline problem report - generation %s\n" generation)(flush)
     (println "Test total error for best:" best-total-test-error)
-    (println (format "Test mean error for best: %.5f" (double (/ best-total-test-error (count best-test-errors)))))
+    (println (format "Test mean error for best: %.5f"
+               (generation-data! [:best :mean-test-error]
+                 (double (/ best-total-test-error (count best-test-errors))))))
     (when (zero? (:total-error best))
       (doseq [[i error] (map vector
                              (range)