Browse files

First steps of core.typed integration.

  • Loading branch information...
Gonzih committed Aug 30, 2013
1 parent 0aad908 commit 1c41d814bdb054d57e644013c85275ec9a45a114
@@ -8,5 +8,6 @@
[org.clojars.scsibug/feedparser-clj "0.4.0"]
[hiccup "1.0.3"]
[org.clojure/tools.logging "0.2.6"]
[log4j/log4j "1.2.17"]]
[log4j/log4j "1.2.17"]
[org.clojure/core.typed "0.2.1"]]
:main feeds2imap.core)
@@ -6,9 +6,11 @@
[feeds2imap.folder :as folder]
[feeds2imap.macro :refer :all]
[ :refer [info error]]
[clojure.pprint :refer [pprint]])
[clojure.pprint :refer [pprint]]
[clojure.core.typed :refer :all])
(:import [ NoRouteToHostException UnknownHostException]
[javax.mail MessagingException]))
[javax.mail MessagingException]
[clojure.lang Keyword]))
(set! *warn-on-reflection* true)
@@ -28,6 +30,7 @@
(catch* [UnknownHostException NoRouteToHostException MessagingException] e
(info "Exception in pull" e)))))
(ann sleep [Long -> nil])
(defn sleep [ms]
(Thread/sleep ms))
@@ -36,6 +39,7 @@
(sleep (* 60 60 1000))
(ann add [Keyword String -> Any])
(defn add [folder url]
(let [folder (keyword folder)
urls (settings/urls)
@@ -5,21 +5,32 @@
[clojure.string :as s]
[clojure.pprint :refer :all]
[ :refer [info error]]
[feeds2imap.macro :refer :all])
[feeds2imap.macro :refer :all]
[clojure.core.typed :refer :all]
[feeds2imap.types :refer :all])
(:import [ MessageDigest]
[ NoRouteToHostException ConnectException UnknownHostException]
[ IOException]))
[ IOException]
[javax.mail Session]
[javax.mail.internet MimeMessage]
[clojure.lang Keyword]))
(ann parse-feed [String -> ParsedFeed])
(ann map-items (Fn [(Fn [ParsedFeed -> Items]) (Folder ParsedFeed) -> (Folder UnflattenedItems)]
[(Fn [Item -> Message]) (Folder Items) -> (Folder Messages)]))
(defn ^:private map-items
"Map function over items for each folder."
[fun coll]
(map (fn [[folder items]] [folder (map fun items)]) coll))
(ann pmap-items [(Fn [String -> ParsedFeed]) (Folder Urls) -> (Folder ParsedFeed)])
(defn ^:private pmap-items
"Map function over items for each folder using pmap."
[fun coll]
(pmap (fn [[folder items]] [folder (pmap fun items)]) coll))
(ann filter-items [(Fn [Item -> Boolean]) (Folder Items) -> (Folder Items)])
(defn ^:private filter-items
"Filter items for each folder.
Filter folders with non empty items collection."
@@ -30,11 +41,13 @@
(filter (fn [[folder items]]
(seq items)))))
(defn ^:private flattern-items [items]
(map (fn [[folder emails]]
[folder (flatten emails)])
(ann flatten-items [(Folder UnflattenedItems) -> (Folder Items)])
(defn ^:private flatten-items [items]
(map (fn [[folder items]]
[folder (flatten items)])
(ann item-authors [Item -> String])
(defn ^:private item-authors [{:keys [authors]}]
(s/join (map (fn [author]
(str (:name author)
@@ -43,21 +56,27 @@
" | "
(:uri author))) authors)))
(defn md5 [string]
(non-nil-return MessageDigest/GetInstance :all)
(ann md5 [String -> String])
(defn md5 [^String string]
{:pre [(string? string)]}
(let [md (MessageDigest/getInstance "MD5")]
(.toString (BigInteger. 1 (.digest md (.getBytes string "UTF-8"))) 16)))
(str (.toString (BigInteger. 1 (.digest md (.getBytes string "UTF-8"))) 16))))
(ann digest [Item -> String])
(defn ^:private digest
"Generates unique digest for item."
[{:keys [title link] :as item}]
(md5 (str title link (item-authors item))))
(ann new? [Cache Item -> Boolean])
(defn ^:private new?
"Looks up item in the cache"
[cache item]
(not (contains? cache (digest item))))
(ann mark-all-as-read [Cache Items -> Cache])
(defn mark-all-as-read
"Adds all items to cache.
Returns updated cache."
@@ -68,28 +87,39 @@
(map digest)
(into cache)))
;TODO use hiccup here, extract all data from item
(ann item-content [Item -> String])
(defn item-content [item]
(str (or (-> item :contents first :value)
(-> item :description :value))))
(ann to-email-map [String String Item -> (HMap :mandatory {:from String
:to String
:subject String
:html String})])
(defn to-email-map
"Convert item to map for Message construction."
[from to item]
(let [{:keys [title link]} item
authors (item-authors item)
content (-> item :contents first :value)
content (item-content item)
html (html [:table
[:tbody [:tr [:td [:a {:href link} title] [:hr]]]
(when (seq authors)
[:tr [:td authors [:hr]]])
[:tr [:td content]]]])]
{:from from :to to :subject title :html html}))
(ann items-to-email [Session String String Item -> Message])
(defn items-to-emails [session from to item]
(message/from-map session (to-email-map from to item)))
(ann to-emails [Session String String (Folder Items) -> (Folder Messages)])
(defn to-emails
"Convert items to Messages."
[session from to items]
(map-items (partial items-to-emails session from to) items))
(map-items (partial items-to-emails session from to) items)) ; (Folder Messages)
(ann parse [String -> ParsedFeed])
(defn parse [url]
(letfn [(log-try [url n-try reason]
(if (> n-try 1)
@@ -109,9 +139,10 @@
IOException] e (parse-try url (inc n-try) e)))))]
(parse-try url)))
(ann new-items [Cache (Folder Urls) -> (Folder Items)])
(defn new-items [cache urls]
(->> urls
(pmap-items parse)
(map-items :entries)
(filter-items (partial new? cache))))
(filter-items (partial new? cache)))) ;items
@@ -1,22 +1,32 @@
(ns feeds2imap.folder
(:require [ :refer [info error]])
(:import [javax.mail Folder Message]))
(:require [ :refer [info error]]
[clojure.core.typed :refer :all])
(:import [javax.mail Store Folder Message]))
(defn get-folder [store folder]
(non-nil-return javax.mail.Store/getFolder :all)
(non-nil-return javax.mail.Folder/exists :all)
(non-nil-return javax.mail.Folder/create :all)
(ann get-folder [Store String -> Folder])
(defn ^Folder get-folder [^Store store ^String folder]
(.getFolder store folder))
(defn exists [store folder]
(ann exists [Store String -> Boolean])
(defn exists [^Store store folder]
(.exists (get-folder store folder)))
(defn create [store folder]
(ann create [Store String -> (U Boolean nil)])
(defn create [^Store store folder]
(when-not (exists store folder)
(info "Creating IMAP folder" folder)
(.create (get-folder store folder) Folder/HOLDS_MESSAGES)))
(ann append [Store String (Vec Message) -> nil])
(defn append [store folder messages]
(.appendMessages (get-folder store folder)
(into-array Message messages)))
(ann append-emails [Store (Vec Message) -> (Seq Message)])
(defn append-emails [store emails]
(pmap (fn [[folder emails]]
@@ -1,21 +1,29 @@
(ns feeds2imap.imap
(:require [feeds2imap.folder :as folder]
[feeds2imap.message :as message])
(:import [javax.mail Session Store]
[feeds2imap.message :as message]
[clojure.core.typed :refer :all])
(:import [javax.mail Session Store Authenticator]
[java.util Properties]))
(defn get-props []
(non-nil-return javax.mail.Session/getStore :all)
(non-nil-return javax.mail.Session/getInstance :all)
(ann get-props [-> Properties])
(defn ^Properties get-props []
(doto (Properties.)
(.put "" "imap")
(.put "mail.imap.starttls.enable" "true")
(.put "mail.imap.socketFactory.class" "")))
(defn get-session [props authenticator]
(ann get-session [Properties Authenticator -> Session])
(defn ^Session get-session [^Properties props authenticator]
(Session/getInstance props authenticator))
(defn ^Store get-store [session]
(ann get-store [Session -> Store])
(defn ^Store get-store [^Session session]
(.getStore session))
(defn connect [store host port username password]
(ann connect [Store String int String String -> Store])
(defn ^Store connect [^Store store host port username password]
(.connect store host port username password)
@@ -1,15 +1,30 @@
(ns feeds2imap.message
(:import [javax.mail Message$RecipientType]
(:require [clojure.core.typed :refer :all]
[feeds2imap.types :refer :all])
(:import [javax.mail Message$RecipientType Session]
[javax.mail.internet MimeMessage InternetAddress MimeBodyPart]
[java.util Date]))
(defn from-map
(non-nil-return javax.mail.Message$RecipientType/TO :all)
(ann recipient-type-to [-> Message$RecipientType])
(defn ^Message$RecipientType recipient-type-to
{:post [%]}
(ann from-map [Session (HMap :mandatory {:from String
:to String
:subject String
:html String })
-> Message])
(defn ^MimeMessage from-map
"Create message from map."
[session {:keys [from to subject html]}]
[^Session session {:keys [from ^String to subject html]}]
(let [message (MimeMessage. session)]
(doto message
(.setFrom (InternetAddress. from))
(.setRecipients Message$RecipientType/TO to)
(.setRecipients (recipient-type-to) to)
(.setSubject subject)
(.setContent html "text/html; charset=utf-8")
(.setSentDate (Date.)))))
@@ -1,43 +1,61 @@
(ns feeds2imap.settings
(:require [clojure.edn :as edn]
[ :as io]
[ :refer [info error]])
(:import [ File]))
[ :refer [info error]]
[clojure.core.typed :refer :all]
[feeds2imap.types :refer :all])
(:import [ File]
[clojure.lang Keyword]))
(ann config-dir [-> String])
(defn ^:private config-dir []
(str (System/getenv "HOME") "/.config/feeds2imap.clj/"))
(ann file [String -> File])
(defn ^File file [^String path]
(File. path))
(ann bootsrap-config-dir [-> Any])
(defn ^:private bootstrap-config-dir []
(let [file (File. (config-dir))]
(let [file (file (config-dir))]
(when-not (.exists file)
(.mkdirs file))))
(ann bootstrap-file [String Any & :optional {:force Boolean} -> Any])
(defn ^:private bootstrap-file [path initial & {:keys [force] :or {force false}}]
(let [file (File. path)]
(let [file (file path)]
(when (or force (not (.exists file)))
(.createNewFile file)
(spit path (str initial)))))
(ann read-or-create-file (Fn [String (Set String) -> Cache]
[String (HMap) -> (Folder Urls)]))
(defn ^:private read-or-create-file [path initial]
(let [path (str (config-dir) path)]
(bootstrap-file path initial)
(edn/read-string (slurp path))))
(ann write-file [String (U Cache (Folder Urls)) -> Any])
(defn ^:private write-file [path data]
(bootstrap-file (str (config-dir) path) data :force true))
(ann read-items [-> (Set String)])
(defn read-items []
(read-or-create-file "read-items.clj" (hash-set)))
(ann write-items [Cache -> Any])
(defn write-items [data]
(info "Writing" (count data) "items to cache.")
(write-file "read-items.clj" data))
(ann imap [-> (Folder Urls)])
(defn imap []
(read-or-create-file "imap.clj" (hash-map)))
(ann urls (Fn [-> (Folder Urls)]
[(Folder Urls) -> Any]))
(defn urls
([] (read-or-create-file "urls.clj" (hash-map)))
([data] (write-file "urls.clj" data)))
@@ -0,0 +1,27 @@
(ns feeds2imap.types
(:require [clojure.core.typed :refer :all])
(:import [clojure.lang Keyword]
[javax.mail.internet MimeMessage]))
(def-alias Cache (Set String))
(def-alias Item
(HMap :mandatory {:authors (Seqable String)
:title String
:link String
:contents (Seqable (HMap :optional {:value String}))
:description (HMap :optional {:value String})}))
(def-alias Items (Seqable Item))
(def-alias UnflattenedItems (Seqable Items))
(def-alias ParsedFeed
(HMap :mandatory {:entries Items}))
(def-alias Message MimeMessage)
(def-alias Messages (Seqable Message))
(def-alias Urls (Vec String))
(def-alias Folder
(TFn [[x :variance :covariant]] (Map Keyword x)))

0 comments on commit 1c41d81

Please sign in to comment.