forked from dbamman/latin-texts
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
133 lines (93 loc) · 5.87 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
SHELL := /bin/bash
DIRS := djvus latin_txt greatbooks djvu_xmls djvu_xml_txts ocr-results
TXT_PREFIXES := $(foreach dir,$(DIRS),$(dir)-language-confidence $(dir)-latin-confidence $(dir)-combined-confidence $(dir)-combined-confidence-diff $(dir)-lines)
TXT_SORTED := $(addsuffix -sorted,$(TXT_PREFIXES))
TXT_TARGETS := $(addsuffix .txt,$(TXT_PREFIXES) $(TXT_SORTED)) latin_to_annotate.tsv missing_ids_200_pdfs.txt missing_ids_200_no_pdfs.txt missing_ids_200_xmls.txt missing_ids_200_no_xmls.txt
TXT_FILES := language-confidence.txt latin-confidence.txt language-confidence-sorted.txt latin-confidence-sorted.txt combined-confidence.txt combined-confidence-sorted.txt
all: $(TXT_TARGETS)
print-% : ; @echo $* = $($*)
%-language-confidence.txt: %
find $^ -type f | langid -b | dos2unix > $@
%-latin-confidence.txt: %
find $^ -type f | langid -l la -b | dos2unix > $@
%-lines.txt: %
find $^ -type f -exec wc -l {} \; | awk '{print $$1 "," $$2}' > $@
%-lines-sorted.txt: %-lines.txt
sort -n < $^ > $@
%-combined-confidence-sorted.txt: %-combined-confidence.txt
awk -F, '{print $$6 "," $$2 "," $$3 "," $$1 "," $$8}' < $^ | sort -g > $@
%-confidence-sorted.txt: %-confidence.txt
awk -F, '{print $$3 "," $$2 "," $$1}' < $^ | sort -g > $@
%-combined-confidence.txt: %-language-confidence.txt %-latin-confidence.txt %-lines.txt
paste -d, <(sort < $(word 1,$^)) <(sort < $(word 2,$^)) <(awk -F, '{print $$2 "," $$1}' < $(word 3,$^) | sort) > $@
%-combined-confidence-diff.txt: %-combined-confidence-sorted.txt
awk -F, '{diff = sprintf("%.3f",($$1 - $$3) / $$5); print diff "," $$5 "," $$1 "," $$2 "," $$3 "," $$4 }' < $^ > $@
%-combined-confidence-diff-sorted.txt: %-combined-confidence-diff.txt
sort -gr -t, -k1,1 -k2,2 < $^ > $@
ids.txt: Makefile.ids
make -f Makefile.ids $@
latin_to_annotate_not_missing.txt: latin_to_annotate.txt ids.txt
awk 'FNR==NR{a[$$0]=$$0;next} FNR!=NR{ if ($$1 in a) print }' ids.txt latin_to_annotate.txt > $@
latin_to_annotate_missing.txt: latin_to_annotate.txt ids.txt
awk 'FNR==NR{a[$$0]=$$0;next} FNR!=NR{ if (($$1 in a == 0) && (FNR > 1)) print }' ids.txt latin_to_annotate.txt > $@
djvus-original-merged.txt: ids.txt latin_to_annotate_not_missing.txt djvus-combined-confidence-diff-sorted.txt
paste -d, <(sort -t, -k6,6 < djvus-combined-confidence-diff-sorted.txt) <(sort < latin_to_annotate_not_missing.txt) > $@
latin_to_annotate_header.csv: latin_to_annotate.txt
head -n 1 $^ | tr "\\t" , | sed -e 's/$$/,langid/' > $@
latin_to_annotate_missing.csv: latin_to_annotate_missing.txt
tr "\\t" , < $^ | sed -e 's/$$/,/' > $@
djvus-original-merged-sorted.csv: djvus-original-merged.txt latin_to_annotate_missing.txt
sort -gr -t, -k1,1 -k2,2 < djvus-original-merged.txt|tr "\\t" ","|awk -F, '{print $$7 ",," $$9 ",,,," $$13 "," $$4}' > $@
latin_to_annotate.tsv: latin_to_annotate_header.csv djvus-original-merged-sorted.csv latin_to_annotate_missing.csv
cat $^ | tr , "\\t" > $@
djvus: ids.txt $(shell make -s -f Makefile.ids; sed -e 's/$$/_djvu.txt/' -e 's/^/djvus\//' < ids.txt)
touch djvus
djvus/%_djvu.txt:
wget -P djvus "https://archive.org/download/$(patsubst djvus/%_djvu.txt,%,$@)/$(patsubst djvus/%_djvu.txt,%_djvu.txt,$@)"
greatbooks: greatbooks.txt $(shell sed -e 's/$$/_djvu.txt/' -e 's/^/greatbooks\//' < greatbooks.txt)
touch greatbooks
greatbooks/%_djvu.txt:
wget -P greatbooks "https://archive.org/download/$(patsubst greatbooks/%_djvu.txt,%,$@)/$(patsubst greatbooks/%_djvu.txt,%_djvu.txt,$@)"
gdown.pl:
wget 'https://raw.githubusercontent.com/Nanolx/patchimage/master/tools/gdown.pl'
chmod +x gdown.pl
latin_txt.tar.gz: gdown.pl
./gdown.pl 'https://docs.google.com/uc?id=0B5pGKi0iCsnbZEdHZ3N6d216am8&export=download' $@
latin_txt: latin_txt.tar.gz
tar xzvf $^
touch latin_txt
djvu_xml_txts: djvu_xmls djvu2txt.xsl
rm -rfv djvu_xml_txts
mkdir djvu_xml_txts
saxon -xsl:djvu2txt.xsl -s:djvu_xmls -o:djvu_xml_txts
djvu_xmls: missing_ids_200_xmls.txt
rm -rfv djvu_xmls
mkdir djvu_xmls
while read i; do wget -P djvu_xmls "https://archive.org/download/$${i}/$${i}_djvu.xml"; done < missing_ids_200_xmls.txt
touch djvu_xmls
.PHONY: downloads
downloads:
wget -N -c 'https://github.com/ryanfb/latin-texts-ocr/archive/master.zip'
master.zip: downloads
ocr-results: master.zip
rm -rf ocr-results
unzip master.zip
mv -v latin-texts-ocr-master/ocr-results ocr-results
rm -rfv latin-texts-ocr-master
touch ocr-results
missing_ids_404.txt missing_ids_200.txt: missing_ids.txt
rm -fv missing_ids_404.txt missing_ids_200.txt
while read id; do if curl --retry 5 -s --fail --head -o /dev/null http://archive.org/download/$$id; then echo $$id >> missing_ids_200.txt; else echo $$id >> missing_ids_404.txt; fi; done < missing_ids.txt
missing_ids_200_derivatives.txt missing_ids_200_no_derivatives.txt: missing_ids_200.txt
rm -fv missing_ids_200_derivatives.txt missing_ids_200_no_derivatives.txt
while read id; do if (curl -s -L http://archive.org/download/$${id}/$${id}_files.xml | grep derivative > /dev/null 2>&1); then echo $$id >> missing_ids_200_derivatives.txt; else echo $$id >> missing_ids_200_no_derivatives.txt; fi; done < missing_ids_200.txt
missing_ids_200_pdfs.txt missing_ids_200_no_pdfs.txt: missing_ids_200_derivatives.txt
rm -fv missing_ids_200_pdfs.txt missing_ids_200_no_pdfs.txt
while read id; do if curl --retry 5 -s --fail -L -r 0-0 -o /dev/null http://archive.org/download/$${id}/$${id}.pdf; then echo $$id >> missing_ids_200_pdfs.txt; else echo $$id >> missing_ids_200_no_pdfs.txt; fi; done < missing_ids_200_derivatives.txt
missing_ids_200_xmls.txt missing_ids_200_no_xmls.txt: missing_ids_200_derivatives.txt
rm -fv missing_ids_200_xmls.txt missing_ids_200_no_xmls.txt
while read id; do if curl --retry 5 -s --fail -L -r 0-0 -o /dev/null http://archive.org/download/$${id}/$${id}_djvu.xml; then echo $$id >> missing_ids_200_xmls.txt; else echo $$id >> missing_ids_200_no_xmls.txt; fi; done < missing_ids_200_derivatives.txt
clean:
rm -fv $(TXT_TARGETS)
clean-data:
rm -rfv djvus