-
Notifications
You must be signed in to change notification settings - Fork 11
/
merger.clj
155 lines (138 loc) · 6.15 KB
/
merger.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
(ns stencil.merger
"Token listaban a text tokenekbol kiszedi a parancsokat es action tokenekbe teszi."
(:require [clojure.data.xml :as xml]
[clojure.string :refer [index-of ends-with?]]
[stencil.postprocess.ignored-tag :as ignored-tag]
[stencil
[types :refer [open-tag close-tag]]
[tokenizer :as tokenizer]
[util :refer [prefixes suffixes subs-last string parsing-exception]]]))
(set! *warn-on-reflection* true)
;; only fragment includes are evaluated
(def ^:dynamic *only-includes* false)
(defn peek-next-text
"Returns a lazy seq of text content characters from the token list."
[tokens]
((fn f [stack tokens]
(when-let [[t & ts] (seq tokens)]
(if-let [text (:text t)]
(concat (for [[t & trs] (suffixes text)]
{:char t
:stack stack
:text-rest trs
:rest ts})
(lazy-seq (f stack ts)))
(recur (cons t stack) ts))))
nil tokens))
(defn find-first-code [^String s]
(assert (string? s))
(when-let [ind (index-of s (str open-tag))]
(if-let [after-idx (index-of s (str close-tag) ind)]
(cond-> {:action (subs s (+ ind (count open-tag)) after-idx)}
(pos? ind) (assoc :before (subs s 0 ind))
(not= (+ (count close-tag) after-idx) (count s))
(assoc :after (subs s (+ (count close-tag) after-idx))))
(cond-> {:action-part (subs s (+ ind (count open-tag)))}
(not (zero? ind)) (assoc :before (subs s 0 ind))))))
(defn text-split-tokens [^String s]
(assert (string? s))
(loop [s s
output []]
(if-let [x (some-> s find-first-code)]
(if (:action-part x)
{:tokens (if-let [b (:before x)] (conj output {:text b}) output)
:action-part (:action-part x)}
(recur (:after x)
(if (seq (:before x))
(conj output {:text (:before x)} {:action (:action x)})
(conj output {:action (:action x)}))))
(if (seq s)
{:tokens (conj output {:text s})}
{:tokens output}))))
;; returns a map of {:char :stack :text-rest :rest}
(defn -find-open-tag [last-chars-count next-token-list]
(assert (integer? last-chars-count))
(assert (pos? last-chars-count))
(assert (sequential? next-token-list))
(let [next-text (peek-next-text next-token-list)
n (- (count open-tag) last-chars-count)]
(when (= (drop last-chars-count open-tag)
(take n (map :char next-text)))
(nth next-text (dec n)))))
(defn -last-chars-count [sts-tokens]
(assert (sequential? sts-tokens))
(when-let [last-text (some-> sts-tokens last :text string)]
(some #(when (ends-with? last-text (string %))
(count %))
(prefixes open-tag))))
(defn map-action-token [token]
(if-let [action (:action token)]
(let [parsed (tokenizer/text->cmd action)
parsed (assoc parsed :raw (str open-tag action close-tag))]
(if (and *only-includes*
(not= :cmd/include (:cmd parsed)))
{:text (str open-tag action close-tag)}
{:action parsed}))
token))
(declare cleanup-runs)
(defn cleanup-runs-1 [[first-token & rest-tokens]]
(assert (:text first-token))
(let [sts (text-split-tokens (:text first-token))]
(if (:action-part sts)
;; Ha van olyan akcio resz, amit elkezdtunk de nem irtunk vegig...
(let [next-token-list (cons {:text (:action-part sts)} rest-tokens)
[this that] (split-with #(not= (seq close-tag)
(take (count close-tag) (map :char %)))
(suffixes (peek-next-text next-token-list)))
that (if (empty? that)
(throw (parsing-exception "" (str "Stencil tag is not closed. Reading " open-tag
(string (comp (take 20) (map first) (map :char)) this))))
;; (throw (ex-info "Tag is not closed? " {:read (first this)}))
(first (nth that (dec (count close-tag)))))
; action-content (apply str (map (comp :char first) this))
]
(concat
(map map-action-token (:tokens sts))
(let [ap (map-action-token {:action (string (map (comp :char first)) this)})]
(if (:action ap)
(concat
[ap]
(reverse (:stack that))
(if (seq (:text-rest that))
(lazy-seq (cleanup-runs-1 (cons {:text (string (:text-rest that))} (:rest that))))
(lazy-seq (cleanup-runs (:rest that)))))
(list* {:text (str open-tag (:action-part sts))}
(lazy-seq (cleanup-runs rest-tokens)))))))
;; If the current :text node ends with a prefix of open-tag:
(if-let [last-chars-count (-last-chars-count (:tokens sts))]
(if-let [this (-find-open-tag last-chars-count rest-tokens)]
(concat
(map map-action-token (butlast (:tokens sts)))
(when-let [s (seq (drop-last last-chars-count (:text (last (:tokens sts)))))]
[{:text (apply str s)}])
(let [tail (cleanup-runs-1
(concat [{:text (apply str open-tag (:text-rest this))}]
(reverse (:stack this))
(:rest this)))]
(if (:action (first tail))
tail
(cons {:text (subs-last (:text (last (:tokens sts))) last-chars-count)}
(lazy-seq (cleanup-runs rest-tokens))))))
(concat (map map-action-token (:tokens sts)) (cleanup-runs rest-tokens)))
(concat (map map-action-token (:tokens sts)) (cleanup-runs rest-tokens))))))
(defn cleanup-runs [token-list]
(when-let [[t & ts] (seq token-list)]
(if (:text t)
(cleanup-runs-1 token-list)
(cons t (lazy-seq (cleanup-runs ts))))))
(defn- map-token [token] (:action token token))
(defn parse-to-tokens-seq
"Parses input and returns a token sequence."
[input]
(->> input
(xml/parse)
(ignored-tag/map-ignored-attr)
(tokenizer/structure->seq)
(cleanup-runs)
(map map-token)))
:OK