-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
187 lines (157 loc) · 5.64 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
SHELL := /bin/zsh
sparql := /home/freundt/usr/apache-jena/bin/sparql
stardog := STARDOG_JAVA_ARGS='-Dstardog.default.cli.server=http://plutos:5820' /home/freundt/usr/stardog/bin/stardog
sdexec := $(stardog) query execute -g "http://data.ga-group.nl/un-locode/" -r -l -1 iso
include .release
all: un-locode.ttl
check: check.un-locode check.un-locode-hist
canon: un-locode-align.ttl.canon un-locode-hist.ttl.canon un-locode.ttl.canon
fixup: fixup.un-locode-hist fixup.un-locode
## untar and cat CodeListPart*.csv to this file
dmp = download/release.csv
## use make all fixup check
## fix issues, then
## make decouple canon check
## make replaced canon fixup
tmp/remarks.out: un-locode.ttl
tmp/%.csv: download/%.htm scripts/rdtbl.xsl
xsltproc --html scripts/rdtbl.xsl $< \
> $@.t && mv $@.t $@
tmp/%.ttl: tmp/%.csv sql/mklocode.tarql
tarql -t sql/mklocode.tarql $< \
> $@.t && mv $@.t $@
tmp/codes-locode.ttl: download/codes-locode/vocab/unlocode
find $</ -name '*.jsonld' \
| xargs ~/usr/stardog/bin/stardog file cat -f PRETTY_TURTLE \
| sed 's@unlcdf:functions@un-loc:hasFunction@; s@rdf:value@skos:notation@; s@unlcdc:countryCode[ \t]*unlcdc:@lcc-cr:isPartOf cc1:@; s@unlcds:countrySubdivision[ \t]*unlcds:\(..\)@lcc-cr:isPartOf cc2:\1-@' \
> $@
tmp/un-locode.ttl: .release $(dmp)
scripts/canon.R $(filter-out $<,$^) \
| tarql -t --stdin --base '!$(REV),$(RDT)' sql/mklocode.tarql \
> $@.t && mv $@.t $@
tmp/un-locode.seen: tmp/un-locode.ttl
## snarf old guys
-ttl2ttl --sortable un-locode.ttl \
| grep -vF 'owl:sameAs' \
| cut -f1 \
| sort -u \
> tmp/un-locode.prev
# snarf new guys
ttl2ttl --sortable $< \
| cut -f1 \
| sort -u \
> $@.t && mv $@.t $@
tmp/un-locode-new.ttl: tmp/un-locode.seen
# see who's been added in this release
dtchanges tmp/un-locode.{prev,seen} 1 \
| scripts/bang-new.awk -v REV=$(REV) -v RDT=$(RDT) \
> $@.t && mv $@.t $@
tmp/un-locode-tempo.ttl: .release $(dmp)
# rescue validFroms and efficaciousFroms of active nodes
-ttl2ttl --sortable un-locode.ttl \
| grep $$'^unlcd:.....\t' \
| grep -F $$'tempo:validFrom\ntempo:efficaciousFrom' \
> $@
tmp/un-locode.kick: tmp/un-locode.seen
# see who's been deleted in this release
dtchanges tmp/un-locode.{prev,seen} 1 \
| scripts/grep-old.awk \
> $@.t && mv $@.t $@
tmp/un-locode-hist.ttl: tmp/un-locode.kick tmp/un-locode.seen
# see who's been deleted in this release
dtchanges tmp/un-locode.{prev,seen} 1 \
| scripts/kick-old.awk -v REV=$(REV) -v RDT=$(RDT) \
> $@.t && mv $@.t $@
# copy the bobs from un-locode.ttl
ttl2ttl --sortable un-locode.ttl \
| grep -Ff $< \
| mawk -F '\t' '{$$1=$$1"_$(REV)"}1' \
> $@.t && cat $@.t >> $@
un-locode-hist.ttl: tmp/un-locode-tempo.ttl tmp/un-locode-hist.ttl
cat tmp/un-locode-hist.ttl >> $@
touch $@
$(MAKE) $@.canon
un-locode.ttl: un-locode-aux.ttl tmp/un-locode.ttl tmp/un-locode-new.ttl un-locode-hist.ttl tmp/un-locode-tempo.ttl un-locode-align.ttl
cat $^ \
> $@.t && mv $@.t $@
$(MAKE) $@.canon
un-locode-align.ttl: download/alternateNamesV2.zip
ttl2ttl --sortable un-locode-hist.ttl \
| grep -F owl:sameAs \
> tmp/alias
tar xf $< -O - alternateNamesV2.txt \
| scripts/alignify.awk \
| tarql -t --stdin sql/mkalign.tarql \
| ttl2ttl --sortable \
| tee tmp/align \
| grep -vFf <(cut -f1 tmp/alias) \
> $@.t && mv $@.t $@
scripts/tempoXsameAs.R tmp/ali{gn,as} \
>> $@
$(MAKE) $@.canon
%.ttl.canon: %.ttl
rapper -i turtle $< >/dev/null
ttl2ttl --sortable $< \
| tr '@' '\001' \
| sed 's@rdf:type@a@' \
| sort -u \
| tr '\001' '@' \
| ttl2ttl -B \
> $@ && mv $@ $<
tmp/%.out: sql/%.sql
$(sdexec) --format CSV $< \
> $@.t && mv $@.t $@
tmp/decouple.cand: /tmp/check.un-locode.ttl
grep -A 2 -F 'sameAs-solitude' $< \
| grep -F focusNode \
| mawk -F'/' '$$0="un-loc:"substr($$NF,1,5)' \
> $@.t && mv $@.t $@
tmp/decouple.only: un-locode.ttl tmp/decouple.cand
ttl2ttl --sortable $< \
| grep -Ff tmp/decouple.cand \
| ttl2ttl \
> $@.t && mv $@.t $@
auto.decouple: un-locode-hist.ttl tmp/decouple.cand
ttl2ttl --sortable $< \
| grep -Ff <(mawk '$$0=$$0"\t"' tmp/decouple.cand) \
> .decouple.t && mv .decouple.t .decouple
decouple: .decouple
ttl2ttl --sortable un-locode.ttl \
| { grep -Ff <(scripts/decouple.awk .decouple); : } \
> $@.t
scripts/tempoXsameAs.R --newpred tempo:efficaciousTill $@.t .decouple \
> $@.eftl && $(RM) $@.t
cat $@.eftl >> un-locode-hist.ttl
cat $@.eftl >> un-locode.ttl
$(RM) $@.eftl
ttl2ttl --sortable un-locode-hist.ttl \
| grep -vFf .decouple \
| sed -f <(scripts/redirreplc.awk .decouple) \
> $@.t && mv $@.t un-locode-hist.ttl
ttl2ttl --sortable un-locode.ttl \
| grep -vFf .decouple \
> $@.t && mv $@.t un-locode.ttl
replaced: tmp/remarks.out
scripts/snarf-replaced.R $< \
| tarql -t --stdin sql/replaced.tarql \
| tee -a un-locode.ttl \
>> un-locode-hist.ttl \
&& touch un-locode.ttl
fixup.%: %.ttl
cat $< \
| scripts/fixup-sameAs.awk \
| scripts/fixup-replaced.awk \
> $@.t && mv $@.t $*.ttl
check.%: %.ttl shacl/%.shacl.ttl
truncate -s 0 /tmp/$@.ttl
$(stardog) data add --remove-all -g "http://data.ga-group.nl/un-locode/" iso $< $(ADDITIONAL)
$(stardog) icv report --output-format PRETTY_TURTLE -g "http://data.ga-group.nl/un-locode/" -r -l -1 iso shacl/$*.shacl.ttl \
>> /tmp/$@.ttl || true
$(MAKE) $*.rpt
%.rpt: /tmp/check.%.ttl
$(sparql) --results text --data $< --query sql/valrpt.sql
setup-stardog:
$(stardog)-admin db create -o reasoning.sameas=OFF -n iso
$(stardog) namespace add --prefix un-loc --uri http://data.ga-group.nl/un-locode/ iso
unsetup-stardog:
$(stardog)-admin db drop iso