forked from documentcloud/cloud-crowd
-
Notifications
You must be signed in to change notification settings - Fork 0
/
process_pdfs.rb
92 lines (80 loc) · 3.15 KB
/
process_pdfs.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# Depends on working pdftk, gm (GraphicsMagick), and pdftotext (Poppler) commands.
# Splits a pdf into batches of N pages, creates their thumbnails and icons,
# as specified in the Job options, gets the text for every page, and merges
# it all back into a tar archive for convenient download.
#
# See <tt>examples/process_pdfs_example.rb</tt> for more information.
class ProcessPdfs < CloudCrowd::Action
# Split up a large pdf into single-page pdfs. Batch them into 'batch_size'
# chunks for processing. The double pdftk shuffle fixes the document xrefs.
def split
`pdftk #{input_path} burst output "#{file_name}_%05d.pdf_temp"`
FileUtils.rm input_path
pdfs = Dir["*.pdf_temp"]
pdfs.each {|pdf| `pdftk #{pdf} output #{File.basename(pdf, '.pdf_temp')}.pdf`}
pdfs = Dir["*.pdf"]
batch_size = options['batch_size']
batches = (pdfs.length / batch_size.to_f).ceil
batches.times do |batch_num|
tar_path = "#{sprintf('%05d', batch_num)}.tar"
batch_pdfs = pdfs[batch_num*batch_size...(batch_num + 1)*batch_size]
`tar -czf #{tar_path} #{batch_pdfs.join(' ')}`
end
Dir["*.tar"].map {|tar| save(tar) }
end
# Convert a pdf page into different-sized thumbnails. Grab the text.
def process
`tar -xzf #{input_path}`
FileUtils.rm input_path
cmds = []
generate_images_commands(cmds)
generate_text_commands(cmds)
system cmds.join(' && ')
FileUtils.rm Dir['*.pdf']
`tar -czf #{file_name}.tar *`
save("#{file_name}.tar")
end
# Merge all of the resulting images, all of the resulting text files, and
# the concatenated merge of the full-text into a single tar archive, ready to
# for download.
def merge
input.each do |batch_url|
batch_path = File.basename(batch_url)
download(batch_url, batch_path)
`tar -xzf #{batch_path}`
FileUtils.rm batch_path
end
names = Dir['*.txt'].map {|fn| fn.sub(/_\d+(_\w+)?\.txt\Z/, '') }.uniq
dirs = names.map {|n| ["#{n}/text/full", "#{n}/text/pages"] + options['images'].map {|i| "#{n}/images/#{i['name']}" } }.flatten
FileUtils.mkdir_p(dirs)
Dir['*.*'].each do |file|
ext = File.extname(file)
name = file.sub(/_\d+(_\w+)?#{ext}\Z/, '')
if ext == '.txt'
FileUtils.mv(file, "#{name}/text/pages/#{file}")
else
suffix = file.match(/_([^_]+)#{ext}\Z/)[1]
sans_suffix = file.sub(/_([^_]+)#{ext}\Z/, ext)
FileUtils.mv(file, "#{name}/images/#{suffix}/#{sans_suffix}")
end
end
names.each {|n| `cat #{n}/text/pages/*.txt > #{n}/text/full/#{n}.txt` }
`tar -czf processed_pdfs.tar *`
save("processed_pdfs.tar")
end
private
def generate_images_commands(command_list)
Dir["*.pdf"].each do |pdf|
name = File.basename(pdf, File.extname(pdf))
options['images'].each do |i|
command_list << "gm convert #{i['options']} #{pdf} #{name}_#{i['name']}.#{i['extension']}"
end
end
end
def generate_text_commands(command_list)
Dir["*.pdf"].each do |pdf|
name = File.basename(pdf, File.extname(pdf))
command_list << "pdftotext -enc UTF-8 -layout -q #{pdf} #{name}.txt"
end
end
end