Merge branch 'deployment'

bin/librato_report.sh

@@ -0,0 +1,11 @@
+curl -u $LIBRATO_EMAIL:$LIBRATO_TOKEN \
+     -d '&gauges[0][name]=CPU'\
+     '&gauges[0][value]='`echo $(ps aux | awk '{s=s+$3}; END{print s}')`''\
+     '&gauges[0][source]=clojurians-log'\
+     '&gauges[1][name]=MEM'\
+     '&gauges[1][value]='`echo $(ps aux | awk '{s=s+$4}; END{print s}')`''\
+     '&gauges[1][source]=clojurians-log' \
+     '&gauges[1][name]=Disk'\
+     '&gauges[2][value]='`df | grep /dev/vda1 | awk '{print $5}' | sed s/%//`''\
+     '&gauges[2][source]=clojurians-log' \
+     -X POST https://metrics-api.librato.com/v1/metrics;

bin/nginx_report.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+require 'time'
+require 'net/http'
+require 'json'
+
+ENDPOINT = 'https://metrics-api.librato.com/v1/metrics'
+
+def read_log(path)
+  IO.read(path).split("\n").map { |l| d,*r = l.split(" - "); [Time.parse(d), *r] rescue nil }.compact
+end
+
+def log_since(log, time)
+  log.select { |l| l[0] >= time }
+end
+
+def histo(values)
+  values.each.with_object(Hash.new(0)) do |v, h|
+    h[v]+=1
+  end
+end
+
+def POST(uri, headers = {})
+  uri = URI(uri)
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = true if uri.scheme == 'https'
+  request = Net::HTTP::Post.new(uri.request_uri, headers)
+  yield request
+  response = http.request(request)
+end
+
+def req_time_bucket(t)
+  if t > 10
+    "10s+"
+  else
+    "#{t.round}s"
+  end
+end
+
+loop do
+  log = read_log('/var/log/nginx/access.log')
+  log = log_since(log, Time.now - 60)
+
+  unless log.empty?
+    codes = histo(log.map{|l| l[2]})
+    request_times = histo(log.map{|l| req_time_bucket(l[3].to_f)})
+
+    codes = codes.map { |k,v| ["HTTP_#{k}", v] }.to_h
+    request_times = request_times.map { |k,v| ["req_time_#{k}", v]}.to_h
+
+    payload = codes.merge(request_times).map.with_index do |(name,count), idx|
+      {"gauges[#{idx}][name]" => name,
+       "gauges[#{idx}][value]" => count,
+       "gauges[#{idx}][source]" => 'clojurians-log'
+      }
+    end.reduce(&:merge)
+
+    p payload
+
+    res = POST(ENDPOINT, {'Content-Type' => 'application/x-www-form-urlencoded'}) do |req|
+      req.basic_auth ENV['LIBRATO_EMAIL'], ENV['LIBRATO_TOKEN']
+      req.set_form_data(payload)
+    end
+
+    p res
+    p res.body
+  end
+
+  sleep 60
+end

bin/sync_logs

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+cd /var/clojure_app
+cp rtmbot/logs/* logs/logs
+cd logs
+./sync
+{
+  echo "(require 'clojurians-log.repl)"
+  echo "(load-slack-data!)"
+  git diff-tree --no-commit-id --name-only -r HEAD | grep logs/201 | xargs -n 1 printf '(clojurians-log.repl/load-log-file! (clojure.java.io/file "'$(pwd)'/%s"))\n'
+} | nc localhost 50505
+sudo systemctl restart clojure_app

resources/public/robots.txt

@@ -0,0 +1,11 @@
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: SemrushBot-SA
+Disallow: /
+
+User-agent: dotbot
+Disallow: /
+
+User-agent: linkdexbot/2.0
+Disallow: /

src/clj/clojurians_log/config.clj

@@ -16,9 +16,7 @@
             :nested true
             :keywordize true}
    :cookies true
-   :session {:store session-store
-             :flash true
-             :cookie-attrs {:http-only true :same-site :strict}}
+   :session false
    :security {:anti-forgery true
               :xss-protection {:enable? true :mode :block}
               :frame-options :sameorigin

src/clj/clojurians_log/db/queries.clj

@@ -29,6 +29,7 @@
            :message/ts
            :message/thread-ts
            {:message/user [:user/name
+                           :user/slack-id
                            :user-profile/image-48]}])
     ...])
 
@@ -123,16 +124,19 @@
        (map assoc-inst)
        (sort-by :message/inst)))
 
-(comment
-
- (let [channel "datomic"
-       day "2015-06-04"
-       messages (time-util/time-with-label "channel-day-messages" (channel-day-messages (user/db) channel day))
-       thread-msgs1 (time-util/time-with-label "thread-messages" (channel-thread-messages-of-day (user/db) channel day))
-       thread-msgs2 (time-util/time-with-label "thread-messages-fast" (thread-messages (user/db)
-                                                                                       (map #(:message/ts %) messages)))]
-   (println "result1 count:" (count thread-msgs1))
-   (println "result2 count:" (count thread-msgs2))
-   )
-
- )
+(defn channel-id-map [db]
+  (into {}
+        (d/q '[:find ?slack-id ?chan
+               :where
+               [?chan :channel/slack-id ?slack-id]]
+             db)))
+
+(doseq [v [#'clojurians-log.db.queries/user-names
+           #'clojurians-log.db.queries/channel-thread-messages-of-day
+           #'clojurians-log.db.queries/channel
+           #'clojurians-log.db.queries/channel-id-map
+           #'clojurians-log.db.queries/channel-list
+           #'clojurians-log.db.queries/channel-days
+           #'clojurians-log.db.queries/channel-day-messages
+           #'datomic.api/db]]
+  (alter-var-root v (fn [f] (memoize f))))

src/clj/clojurians_log/repl.clj

@@ -4,11 +4,13 @@
   instance, or sys admins seeding the production system with data."
   (:require [clojurians-log.application :as app]
             [clojurians-log.slack-api :as slack]
+            [clojurians-log.db.queries :as q]
             [clojurians-log.db.import :as import]
             [clojurians-log.data :as data]
             [clojure.java.io :as io]
             [datomic.api :as d]
-            [clojure.tools.reader.edn :as edn]))
+            [clojure.tools.reader.edn :as edn]
+            [clojure.string :as str]))
 
 (defn read-edn [filepath]
   (-> filepath
@@ -32,7 +34,13 @@
   "Import Slack users and Channels."
   []
   (slack/import-users! (conn))
-  (slack/import-channels! (conn)))
+  (let [channel->db-id (q/channel-id-map (d/db (conn)))
+        channels       (mapv import/channel->tx (slack/channels))]
+    (d/transact (conn) (map (fn [{slack-id :channel/slack-id :as ch}]
+                              (if-let [db-id (channel->db-id slack-id)]
+                                (assoc ch :db/id db-id)
+                                ch))
+                            channels))))
 
 (defn log-files
   "List all files in the given log directory.
@@ -50,7 +58,7 @@
 
 (defn load-log-file! [file]
   (println (str file))
-  (let [msgs (data/event-seq file)
+  (let [msgs (filter #(= (:type %) "message") (data/event-seq file))
         events (keep import/event->tx msgs)]
     (doseq [event events]
       @(d/transact-async (conn) [event]))))
@@ -93,12 +101,20 @@
       (println "Import messages")
       (run! load-log-file! (log-files directory)))))
 
+(defn load-from [date]
+  (->> (log-files)
+       (drop-while #(not (str/starts-with? (.getName %) date)))
+       (run! load-log-file!)))
+
 (comment
-  ;; nc localhost 50505
+  ;; rlwrap nc localhost 50505
   (use 'clojurians-log.repl)
   (load-slack-data!)
+  (load-from "2016-08-04")
   (run! load-log-file! (log-files))
 
+
+
   (load-demo-data! "/home/arne/github/clojurians-log-demo-data")
 
 

src/clj/clojurians_log/views.clj

@@ -145,24 +145,25 @@
   [{:data/keys [usernames channel date hostname] :as context}
    {:message/keys [user inst user text thread-ts ts] :as message}]
 
-  (let [{:user/keys [name]
+  (let [{:user/keys [name slack-id]
          :user-profile/keys [image-48]} user]
 
-      ;; things in the profile
-      ;; :image_512 :email :real_name_normalized :image_48 :image_192 :real_name :image_72 :image_24
-      ;; :avatar_hash :title :team :image_32 :display_name :display_name_normalized
-      (list [:div.message
-             {:id (cl.tu/format-inst-id inst) :class (when (thread-child? message) "thread-msg")}
-             [:a.message_profile-pic {:href "" :style (str "background-image: url(" image-48 ");")}]
-             [:a.message_username {:href ""} name]
-             [:span.message_timestamp [:a {:href (bidi/path-for routes
-                                                                :log-target-message
-                                                                :channel (:channel/name channel)
-                                                                :date date
-                                                                :ts ts)}
-                                       (cl.tu/format-inst-time inst)]]
-             [:span.message_star]
-             [:span.message_content [:p (slack-messages/message->hiccup text usernames)]]])))
+    ;; things in the profile
+    ;; :image_512 :email :real_name_normalized :image_48 :image_192 :real_name :image_72 :image_24
+    ;; :avatar_hash :title :team :image_32 :display_name :display_name_normalized
+    (list [:div.message
+           {:id (cl.tu/format-inst-id inst) :class (when (thread-child? message) "thread-msg")}
+           [:a.message_profile-pic {:href (str "https://clojurians.slack.com/team/" slack-id) :style (str "background-image: url(" image-48 ");")}]
+           [:a.message_username {:href (str "https://clojurians.slack.com/team/" slack-id)} name]
+           [:span.message_timestamp [:a {:rel "nofollow"
+                                         :href (bidi/path-for routes
+                                                              :log-target-message
+                                                              :channel (:channel/name channel)
+                                                              :date date
+                                                              :ts ts)}
+                                     (cl.tu/format-inst-time inst)]]
+           [:span.message_star]
+           [:span.message_content [:p (slack-messages/message->hiccup text usernames)]]])))
 
 (defn- message-hiccup
   "Returns either a single message hiccup, or if the given message starts a thread,

test/clj/clojurians_log/db/import_test.clj

@@ -0,0 +1,44 @@
+(ns clojurians-log.db.import-test
+  (:require [clojure.test :refer :all]
+            [clojurians-log.db.import :as import]))
+
+(deftest event->tx-test
+  (testing "returns valid datomic transaction data"
+    (is (= #:message{:key     "C0617A8PQ--1517728899.000025"
+                     :ts      "1517728899.000025"
+                     :day     "2018-02-04"
+                     :text    ":smile:"
+                     :channel [:channel/slack-id "C0617A8PQ"]
+                     :user    [:user/slack-id "U051BLM8F"]}
+           (import/event->tx {:source_team "T03RZGPFR"
+                              :text        ":smile:"
+                              :ts          "1517728899.000025"
+                              :user        "U051BLM8F"
+                              :team        "T03RZGPFR"
+                              :type        "message"
+                              :channel     "C0617A8PQ"}))))
+
+  (testing "parses thread info"
+    (is (= {:message/thread-ts   "1517483637.000077"
+            :message/thread-inst #inst "2018-02-01T11:13:57.000-00:00"}
+           (-> (import/event->tx {:source_team     "T03RZGPFR"
+                                  :reply_broadcast true
+                                  :channel         "C03S1L9DN"
+                                  :type            "message"
+                                  :thread_ts       "1517483637.000077"
+                                  :ts              "1517487432.000382"
+                                  :team            "T03RZGPFR"
+                                  :user            "U7THYHQ1F"
+                                  :text            "Wow, thanks!"})
+               (select-keys [:message/thread-ts :message/thread-inst])))))
+
+  (testing "puts threaded messages on the day of the thread"
+    (is (= "2018-02-06"
+           (:message/day (import/event->tx {:thread_ts   "1517932488.000769"
+                                            :source_team "T03RZGPFR"
+                                            :text        "posted on 2018-02-08, replying to a message from 2018-02-06"
+                                            :ts          "1518050759.000174"
+                                            :user        "U8MJBRSR5"
+                                            :team        "T03RZGPFR"
+                                            :type        "message"
+                                            :channel     "C03RZRRMP"}))))))

test/clj/clojurians_log/repl_test.clj

@@ -0,0 +1,17 @@
+(ns clojurians-log.repl-test
+  (:require [clojurians-log.repl :as repl]
+            [clojurians-log.slack-api :as slack]
+            [clojure.test :refer :all]))
+
+
+(deftest load-slack-data!-test
+  (testing "it handles renames"
+    (with-redefs [slack/users (constantly [])
+                  slack/channels (constantly {:id "C03RZGPG1"
+                                              :name "announcements"
+                                              :creator "U03RZGPFT"
+                                              :created 1425231209})]
+      (repl/load-slack-data!)
+      )
+
+    ))

test/clj/clojurians_log/slack_api_test.clj

@@ -0,0 +1,34 @@
+(ns clojurians-log.slack-api-test
+  (:require [clojurians-log.slack-api :as slack]
+            [clojurians-log.db.queries :as queries]
+            [clojurians-log.test-helper :refer [test-conn]]
+            [clojure.test :refer :all]
+            [datomic.api :as d]))
+
+(deftest import-channels!-test
+  (testing "it handles renames"
+    (let [conn (test-conn)]
+      (with-redefs [slack/users (constantly [{:id "U03RZGPFT"}])
+                    slack/channels (constantly [{:id "C03RZGPG1"
+                                                 :name "announcements"
+                                                 :creator "U03RZGPFT"
+                                                 :created 1425231209}])]
+
+        (slack/import-users! conn)
+        (slack/import-channels! conn)
+        (is (= (select-keys (queries/channel (d/db conn) "announcements")
+                            [:channel/slack-id :channel/name])
+               {:channel/slack-id "C03RZGPG1"
+                :channel/name "announcements"})))
+
+      (with-redefs [slack/users (constantly [{:id "U03RZGPFT"}])
+                    slack/channels (constantly [{:id "C03RZGPG1"
+                                                 :name "admin-announcements"
+                                                 :creator "U03RZGPFT"
+                                                 :created 1425231209}])]
+
+        (slack/import-channels! conn)
+        (is (= (select-keys (queries/channel (d/db conn) "admin-announcements")
+                            [:channel/slack-id :channel/name])
+               {:channel/slack-id "C03RZGPG1"
+                :channel/name "admin-announcements"}))))))