Descriptive stats works with instants

techascent · Apr 14, 2020 · 836e51d · 836e51d
1 parent 5f50aa0
commit 836e51d
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 1 deletion.
diff --git a/src/tech/ml/dataset/parse/datetime.clj b/src/tech/ml/dataset/parse/datetime.clj
@@ -223,6 +223,9 @@
     `(.add (as-list ~container) ~parsed-val)
     :instant
     `(.add (as-list ~container) ~parsed-val)
+    :packed-instant
+    `(.add (as-long-list (.backing-store (as-typed-buffer ~container)))
+           (pmath/long ~parsed-val))
     :packed-local-date
     `(.add (as-int-list (.backing-store (as-typed-buffer ~container)))
            (pmath/int ~parsed-val))
@@ -242,6 +245,8 @@
   (case datatype
     :instant
     (compile-time-add-to-container! :instant container parsed-val)
+    :packed-instant
+    (compile-time-add-to-container! :packed-instant container parsed-val)
     :zoned-date-time
     (compile-time-add-to-container! :zoned-date-time container parsed-val)
     :offset-date-time

diff --git a/src/tech/ml/dataset/parse/spreadsheet.clj b/src/tech/ml/dataset/parse/spreadsheet.clj
@@ -74,7 +74,10 @@
                                (dtype/get-datatype container))
             ^List container container]
         (dotimes [idx n-missing]
-          (.add container missing-value)
+          (if (dtype-dt/datetime-datatype? (dtype/get-datatype container))
+            (parse-dt/add-to-container! (dtype/get-datatype container) container
+                                        missing-value)
+            (.add container missing-value))
           (.add missing (+ n-elems idx)))))))
 
 

diff --git a/test/tech/ml/dataset/datetime_test.clj b/test/tech/ml/dataset/datetime_test.clj
@@ -35,3 +35,21 @@
     (is (every? dtype-dt/datetime-datatype?
                 (map dtype/get-datatype
                      (vals (select-keys date-only [:min :mean :max])))))))
+
+
+(deftest stocks-descriptive-stats-2
+  (let [stocks (-> (ds/->dataset "test/data/stocks.csv")
+                   (ds/update-column "date" #(dtype/object-reader
+                                              (dtype/ecount %)
+                                              (fn [idx]
+                                                (-> (% idx)
+                                                    (dtype-dt/unpack-local-date)
+                                                    (dtype-dt/local-date->instant)))
+                                              :instant)))
+        desc-stats (ds/descriptive-stats stocks)
+        date-only (-> (ds/filter-column #(= "date" %) :col-name desc-stats)
+                      (ds/mapseq-reader)
+                      (first))]
+    (is (every? dtype-dt/datetime-datatype?
+                (map dtype/get-datatype
+                     (vals (select-keys date-only [:min :mean :max])))))))