Skip to content
This repository has been archived by the owner on Sep 2, 2020. It is now read-only.

Commit

Permalink
Merge pull request #258 from ifesdjeen/improvement/loading-cache-merge
Browse files Browse the repository at this point in the history 
Cache seen prefixes for configurable time period
  • Loading branch information
@ifesdjeen
ifesdjeen committed Jan 2, 2017
2 parents 12905b2 + d178b2a commit 2901aa0
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 40 deletions.
3 changes: 3 additions & 0 deletions doc/source/concepts.rst
View file
Expand Up @@ -129,6 +129,9 @@ a way of querying them back. There are two implementations of this component ava

- ``cassandra`` stores path-names in cassandra.

Cyanite caches index lookups for 1 minute by default. You can configure cache ttl
by using `cache_ttl_in_ms`.

Cyanite has it's own index extension that helps to build more compact trees in Cassandra
SASI index. It's not necessary to use them, although it's highly advised especially if you
have a lot of metrics.
Expand Down
43 changes: 22 additions & 21 deletions project.clj
View file
Expand Up @@ -8,24 +8,25 @@
:jvm-opts ["-Dclojure.compiler.direct-linking=true"]}}
:main io.cyanite
:plugins [[lein-ancient "0.6.7"]]
:dependencies [[org.clojure/clojure "1.8.0"]
[org.clojure/tools.logging "0.3.1"]
[org.clojure/tools.cli "0.3.3"]
[com.stuartsierra/component "0.3.1"]
[spootnik/unilog "0.7.13"]
[spootnik/uncaught "0.5.3"]
[spootnik/globber "0.4.1"]
[spootnik/reporter "0.1.5"]
[spootnik/signal "0.2.0"]
[org.javassist/javassist "3.20.0-GA"]
[instaparse "1.4.1"]
[cheshire "5.5.0"]
[metrics-clojure "2.6.1"]
[clj-yaml "0.4.0"]
[clj-time "0.11.0"]
[cc.qbits/alia "3.2.0"]
[org.jctools/jctools-core "1.2"]
[com.boundary/high-scale-lib "1.0.6"]
[net.jpountz.lz4/lz4 "1.3"]
[org.xerial.snappy/snappy-java "1.1.2.1"]
[io.netty/netty-all "4.0.34.Final"]])
:dependencies [[org.clojure/clojure "1.8.0"]
[org.clojure/tools.logging "0.3.1"]
[org.clojure/tools.cli "0.3.3"]
[com.stuartsierra/component "0.3.1"]
[spootnik/unilog "0.7.13"]
[spootnik/uncaught "0.5.3"]
[spootnik/globber "0.4.1"]
[spootnik/reporter "0.1.5"]
[spootnik/signal "0.2.0"]
[org.javassist/javassist "3.20.0-GA"]
[instaparse "1.4.1"]
[cheshire "5.5.0"]
[metrics-clojure "2.6.1"]
[clj-yaml "0.4.0"]
[clj-time "0.11.0"]
[com.github.ben-manes.caffeine/caffeine "2.3.4"]
[cc.qbits/alia "3.2.0"]
[org.jctools/jctools-core "1.2"]
[com.boundary/high-scale-lib "1.0.6"]
[net.jpountz.lz4/lz4 "1.3"]
[org.xerial.snappy/snappy-java "1.1.2.1"]
[io.netty/netty-all "4.0.34.Final"]])
54 changes: 35 additions & 19 deletions src/io/cyanite/index/cassandra.clj
View file
@@ -1,4 +1,6 @@
(ns io.cyanite.index.cassandra
(:import [com.github.benmanes.caffeine.cache Caffeine CacheLoader LoadingCache]
[java.util.concurrent TimeUnit])
(:require [com.stuartsierra.component :as component]
[io.cyanite.store.cassandra :as c]
[io.cyanite.index :as index]
Expand All @@ -8,6 +10,8 @@
[clojure.string :refer [index-of join split]]
[clojure.tools.logging :refer [error]]))

(def default-cache-ttl-in-ms 60000)

(defn mk-insert-segmentq
[session]
(alia/prepare
Expand Down Expand Up @@ -73,8 +77,7 @@
" AND pos = " (count parts))
;; Prefix wildcard query, (`abc.*.def`), can't use position
:else (str "pos = " (count parts)
" AND segment LIKE '" globbed "' ALLOW FILTERING" )))
)))
" AND segment LIKE '" globbed "' ALLOW FILTERING" ))))))

(defn with-cyanite-tokenizer
[session pattern parts]
Expand All @@ -90,18 +93,39 @@
;; Prefix wildcard query (`abc.*` and alike), add parent
:else (str "SELECT * FROM segment WHERE segment LIKE '" pattern "' AND pos = " pos " ALLOW FILTERING")))))

(defrecord CassandraIndex [options session
(defn load-prefixes-fn
[session index-query-fn pattern]
(let [parts (split pattern #"\.")
pos (count parts)
res (index-query-fn session pattern parts)
filtered (set (glob pattern (map :segment res)))]
(->> res
(filter (fn [{:keys [segment]}]
(not (nil? (get filtered segment)))))
(map (juxt :segment :length :leaf))
(map (partial prefix-info pos)))))

(defrecord CassandraIndex [options session ^LoadingCache cache
insert-segmentq insert-pathq index-query-fn
wrcty rdcty]
component/Lifecycle
(start [this]
(let [[session rdcty wrcty] (c/session! options)]
(-> this
(assoc :session session)
(assoc :index-query-fn (if (:with_tokenizer options)
with-cyanite-tokenizer
native-sasi-index))
(assoc :insert-segmentq (mk-insert-segmentq session)))))
(let [[session rdcty wrcty] (c/session! options)
index-query-fn (if (:with_tokenizer options)
with-cyanite-tokenizer
native-sasi-index)]
(assoc this
:session session
:insert-segmentq (mk-insert-segmentq session)
:cache (-> (Caffeine/newBuilder)
(.expireAfterWrite
(or (:cache_ttl_in_ms options)
default-cache-ttl-in-ms)
TimeUnit/MILLISECONDS)
(.build (reify CacheLoader
(load [this pattern]
(load-prefixes-fn session index-query-fn pattern)))))
)))
(stop [this]
(-> this
(assoc :session nil)
Expand All @@ -121,15 +145,7 @@
(= length i)]
{:consistency wrcty}))))
(prefixes [this pattern]
(let [parts (split pattern #"\.")
pos (count parts)
res (index-query-fn session pattern parts)
filtered (set (glob pattern (map :segment res)))]
(->> res
(filter (fn [{:keys [segment]}]
(not (nil? (get filtered segment)))))
(map (juxt :segment :length :leaf))
(map (partial prefix-info pos))))))
(.get cache pattern)))

(defmethod index/build-index :cassandra
[options]
Expand Down

0 comments on commit 2901aa0

Please sign in to comment.