/
preprocessor.rb
470 lines (399 loc) · 11 KB
/
preprocessor.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
# Copyright (c) 2010-2019 Minero Aoki, Kenshi Muto
# 2002-2009 Minero Aoki
#
# This program is free software.
# You can distribute or modify this program under the terms of
# the GNU LGPL, Lesser General Public License version 2.1.
# For details of the GNU LGPL, see the file "COPYING".
#
require 'review/textutils'
require 'review/exception'
require 'nkf'
module ReVIEW
module ErrorUtils
def init_errorutils(f)
@errutils_file = f
@errutils_err = false
end
def warn(msg)
@logger.warn "#{location}: #{msg}"
end
def error(msg)
@errutils_err = true
raise ApplicationError, "#{location}: #{msg}"
end
def location
"#{filename}:#{lineno}"
end
def filename
@errutils_file.path
end
def lineno
@errutils_file.lineno
end
end
class Preprocessor
include ErrorUtils
def initialize(repo, param)
@repository = repo
@config = param
@logger = ReVIEW.logger
@leave_content = nil
end
def process(inf, outf)
init_errorutils(inf)
@f = outf
begin
preproc(inf)
rescue Errno::ENOENT => e
error e.message
end
end
private
TYPES = %w[file range].freeze
def preproc(f)
init_vars
f.each_line do |line|
case line
when /\A\#@\#/, /\A\#\#\#\#/
@f.print line
when /\A\#@defvar/
@f.print line
direc = parse_directive(line, 2)
defvar(*direc.args)
when /\A\#@mapoutput/
direc = parse_directive(line, 1, 'stderr')
@f.print line
get_output(expand(direc.arg), direc['stderr']).each { |out| @f.print out.string }
skip_list(f)
when /\A\#@mapfile/
direc = parse_directive(line, 1, 'eval')
path = expand(direc.arg)
@leave_content = File.extname(path) == '.re'
if direc['eval']
ent = evaluate(path, ent)
else
ent = @repository.fetch_file(path)
end
replace_block(f, line, ent, false) # FIXME: turn off lineno: tmp
when /\A\#@map(?:range)?/
direc = parse_directive(line, 2, 'unindent')
path = expand(direc.args[0])
@leave_content = File.extname(path) == '.re'
ent = @repository.fetch_range(path, direc.args[1]) or
error "unknown range: #{path}: #{direc.args[1]}"
ent = (direc['unindent'] ? unindent(ent, direc['unindent']) : ent)
replace_block(f, line, ent, false) # FIXME: turn off lineno: tmp
when /\A\#@end/
error 'unbaranced #@end'
when /\A\#@/
op = line.slice(/@(\w+)/, 1)
warn "unknown directive: #{line.strip}" unless known_directive?(op)
if op == 'warn'
warn line.strip.sub(/\#@warn\((.+)\)/, '\1')
end
@f.print line
when /\A\s*\z/ # empty line
@f.puts
else # rubocop:disable Lint/DuplicateBranch
@f.print line
end
end
end
KNOWN_DIRECTIVES = %w[require provide warn ok].freeze
def known_directive?(op)
KNOWN_DIRECTIVES.index(op)
end
def replace_block(f, directive_line, newlines, with_lineno)
@f.print directive_line
newlines.each do |line|
if with_lineno
print_number(line.number)
end
@f.print line.string
end
skip_list(f)
end
def print_number(num)
@f.printf('%4s ', (num ? num.to_s : ''))
end
def skip_list(f)
begline = f.lineno
f.each_line do |line|
case line
when /\A\#@end/
@f.print line
return nil
when %r{\A//\}}
unless @leave_content
warn '//} seen in list'
@f.print line
return nil
end
when /\A\#@\w/
warn "#{line.slice(/\A\#@\w+/)} seen in list"
@f.print line
when /\A\#@/
@f.print line
end
end
error "list reached end of file (beginning line = #{begline})"
end
class Directive
def initialize(op, args, opts)
@op = op
@args = args
@opts = opts
end
attr_reader :op
attr_reader :args
attr_reader :opts
def arg
@args.first
end
def opt
@opts.first
end
def [](key)
@opts[key]
end
end
def parse_directive(line, argc, *optdecl)
m = /\A\#@(\w+)\((.*?)\)(?:\[(.*?)\])?\z/.match(line.strip) or
error "wrong directive: #{line.strip}"
op = m[1]
args = m[2].split(/,\s*/)
opts = parse_optargs(m[3])
return if (argc == 0) && args.empty?
if argc == -1
# Any number of arguments are allowed.
elsif args.size != argc
error 'wrong arg size'
end
if opts
wrong_opts = opts.keys - optdecl
unless wrong_opts.empty?
error "wrong option: #{wrong_opts.keys.join(' ')}"
end
end
Directive.new(op, args, opts || {})
end
def parse_optargs(str)
return nil unless str
table = {}
str.split(/,\s*/).each do |a|
name, spec = a.split('=', 2)
table[name] = optarg_value(spec)
end
table
end
def optarg_value(spec)
case spec
when 'true' # [name=true], [name]
true
when 'false' # [name=false]
false
when 'nil' # [name=nil]
nil
when /^\d+$/ # [name=8]
$&.to_i
else # [name=val]
spec
end
end
def init_vars
@vartable = {}
end
def defvar(name, value)
@vartable[name] = value
end
def expand(str)
str.gsub(/\$\w+/) do |name|
s = @vartable[name.sub('$', '')]
s ? expand(s) : name
end
end
def unindent(chunk, n)
n = minimum_indent(chunk) unless n.is_a?(Integer)
re = /\A#{' ' * n}/
chunk.map { |line| line.edit { |s| s.sub(re, '') } }
end
INF_INDENT = 9999
def minimum_indent(chunk)
n = chunk.map { |line| line.empty? ? INF_INDENT : line.num_indent }.min
n == INF_INDENT ? 0 : n
end
def evaluate(path, chunk)
outputs = get_output("ruby #{path}", false).split(/\n/).map(&:strip)
chunk.map do |line|
if /\# \$\d+/ =~ line.string
# map result into source.
line.edit { |s| s.sub(/\$(\d+)/) { outputs[$1.to_i - 1] } }
else
line
end
end
end
require 'open3'
def get_output(cmd, use_stderr)
out = err = nil
Open3.popen3(cmd) do |_stdin, stdout, stderr|
out = stdout.readlines
if use_stderr
out.concat(stderr.readlines)
else
err = stderr.readlines
end
end
if err && !err.empty?
$stderr.puts '[unexpected stderr message]'
err.each { |line| $stderr.print line }
error 'get_output: got unexpected output'
end
num = 0
out.map { |line| Line.new(num += 1, line) }
end
end
class Line
def initialize(number, string)
@number = number
@string = string
end
attr_reader :number
attr_reader :string
alias_method :to_s, :string
def edit
self.class.new(@number, yield(@string))
end
def empty?
@string.strip.empty?
end
def num_indent
@string.slice(/\A\s*/).size
end
end
class Repository
include TextUtils
include ErrorUtils
def initialize(param)
@repository = {}
@config = param
@logger = ReVIEW.logger
end
def fetch_file(file)
file_descripter(file)['file']
end
def fetch_range(file, name)
fetch(file, 'range', name)
end
def fetch(file, type, name)
table = file_descripter(file)[type] or return nil
table[name]
end
private
def file_descripter(fname)
@leave_content = File.extname(fname) == '.re'
return @repository[fname] if @repository[fname]
@repository[fname] = git?(fname) ? parse_git_blob(fname) : parse_file(fname)
end
def git?(fname)
fname.start_with?('git|')
end
def parse_git_blob(g_obj)
IO.popen('git show ' + g_obj.sub(/\Agit\|/, ''), 'r') do |f|
init_errorutils(f)
return _parse_file(f)
end
end
def parse_file(fname)
File.open(fname, 'rt:BOM|utf-8') do |f|
init_errorutils(f)
return _parse_file(f)
end
end
def _parse_file(f)
whole = []
repo = { 'file' => whole }
curr = { 'WHOLE' => whole }
lineno = 1
yacchack = false # remove ';'-only lines.
opened = [['(not opened)', '(not opened)']] * 3
f.each do |line|
case line
when /(?:\A\#@|\#@@)([a-z]+)_(begin|end)\((.*)\)/
type = check_type($1)
direction = $2
spec = check_spec($3)
case direction
when 'begin'
key = "#{type}/#{spec}"
if curr[key]
error "begin x2: #{key}"
end
(repo[type] ||= {})[spec] = curr[key] = []
when 'end'
curr.delete("#{type}/#{spec}") or
error "end before begin: #{type}/#{spec}"
else
raise 'must not happen'
end
when %r{(?:\A\#@|\#@@)([a-z]+)/(\w+)\{}
type = check_type($1)
spec = check_spec($2)
key = "#{type}/#{spec}"
if curr[key]
error "begin x2: #{key}"
end
(repo[type] ||= {})[spec] = curr[key] = []
opened.push([type, spec])
when %r{(?:\A\#@|\#@@)([a-z]+)/(\w+)\}}
type = check_type($1)
spec = check_spec($2)
curr.delete("#{type}/#{spec}") or
error "end before begin: #{type}/#{spec}"
opened.delete("#{type}/#{spec}")
when /(?:\A\#@|\#@@)\}/
type, spec = opened.last
curr.delete("#{type}/#{spec}") or
error "closed before open: #{type}/#{spec}"
opened.pop
when /(?:\A\#@|\#@@)yacchack/
yacchack = true
when /\A\#@-/ # does not increment line number.
line = canonical($')
curr.each_value { |list| list.push(Line.new(nil, line)) }
else
next if yacchack && (line.strip == ';')
line = canonical(line)
curr.each_value { |list| list.push(Line.new(lineno, line)) }
lineno += 1
end
end
if curr.size > 1
curr.delete('WHOLE')
curr.each { |range, lines| @logger.warn "#{filename}: unclosed range: #{range} (begin @#{lines.first.number})" }
raise ApplicationError, 'ERROR'
end
repo
end
def canonical(line)
if @leave_content
return line
end
tabwidth = @config['tabwidth'] || 8
if tabwidth > 0
detab(line, tabwidth).rstrip + "\n"
else
line
end
end
def check_type(type)
error "wrong type: #{type.inspect}" unless Preprocessor::TYPES.index(type)
type
end
def check_spec(spec)
error "wrong spec: #{spec.inspect}" unless /\A\w+\z/ =~ spec
spec
end
end
end