Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added comments

  • Loading branch information...
commit 6af79b6d2aac7706218bd6988abfbdc74c483fde 1 parent d6d7d54
@julosaure authored
Showing with 11 additions and 3 deletions.
  1. +11 −3 src/SduDataImporter/core.clj
View
14 src/SduDataImporter/core.clj
@@ -12,22 +12,27 @@
(def columnNames ["activity row id" "created date" "primary contact id" "contact integration id" "case" "member" "parish data" "start time" "call type" "type" "inin id" "parish" "interaction time" "resolution" "notes" "division" "program" "task status" "contact external unique id" "subject" "description" "contact type" "caller id" "crmit inin id" "validation status" "worker office number"])
+(def dirToProcess "/Users/julien/Documents/Output/")
+
+(def patFilesToProcess #"LAActivity_AIMA_Part_(.*)\.txt")
+
+;; UTILS
+
(def connection (clojure-solr/connect "http://127.0.0.1:8983/solr"))
(def patDateTime #"(\d{2})/(\d{2})/(\d{4}) (\d{2}:\d{2}:\d{2})")
-(def dirToProcess "/Users/julien/Documents/Output/")
-
-(def patFilesToProcess #"LAActivity_AIMA_Part_(.*)\.txt")
;; FUNCTIONS
(defn changeDateFormat [date]
+ "Change date format from '23-11-2012 03:34:51' to solr format '2012-11-23T03:34:51Z'"
(let [match (re-matches patDateTime date)]
(if match
(str (nth match 3) "-" (nth match 2) "-" (nth match 1) "T" (nth match 4) "Z"))))
(defn lineToMap [line]
+ "Transforms the sequence of fields into a map, and change date fields format"
(let [mapLine (zipmap columnNames (clojure.string/split line columnSeparator))
keys ["created date" "start time"]]
;(println (str mapLine))
@@ -36,6 +41,7 @@
(defn indexLine [line]
+ "Index a line"
(let [mapLine (lineToMap line)]
(if-not (= 0 (compare firstColName (mapLine "activity row id")))
(clojure-solr/with-connection connection
@@ -43,6 +49,7 @@
(defn parseFile [fileName]
+ "Parse a file"
(println (str "Reading file " (.getName fileName)))
(with-open [rdr (clojure.java.io/reader fileName)]
(doseq [line (line-seq rdr)]
@@ -53,6 +60,7 @@
(clojure-solr/commit!)))
(defn parseDir [dirName]
+ "Parse a directory"
(println (str "Reading directory " dirName))
(doseq [f (.listFiles (File. dirName))]
(if (re-matches patFilesToProcess (.getName f))
Please sign in to comment.
Something went wrong with that request. Please try again.