/
stage.sh
executable file
·153 lines (141 loc) · 7.98 KB
/
stage.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#!/usr/bin/env bash
# stage.sh -- a script that stages code and data for distribution
set -eu
###############################################################################
STAGE_DIR=${1:-dist/stage}
stage() {
local src=$1 dst=$2
[[ -e "$src" ]] || {
echo >&2 "$src: No such file to stage"
false
}
dst="$STAGE_DIR/$dst"
dstdir=$(dirname "$dst.")
[[ -d "$dstdir" ]] || mkdir -p "$dstdir"
if [[ -d "$src" ]]; then
# use rsync(1) for staging directories
set -x; rsync -aH --delete "$src" "$dst"
else
# use install(1) for staging files when changed
dstfile=$dst
case $dst in */) dstfile=$dst${src##*/}; esac
if [[ "$src" -nt "$dstfile" ]]; then
if [[ -x "$src" ]]; then
set -x; install -m a=rx "$src" "$dst"
else
# preserve flags of non-executable data since install(1) always marks as executable
set -x; install -m a=r "$src" "$dst"
fi
fi
fi
{ set +x; } &>/dev/null
}
symlink() {
local src=$1 dst=$2
dst="$STAGE_DIR/$dst"
dstdir=$(dirname "$dst.")
[[ -d "$dstdir" ]] || mkdir -p "$dstdir"
case $dst in */) dst=$dst${src##*/}; esac
if ! [[ -L "$dst" && $(readlink "$dst") = $src ]]; then
set -x; ln -sfn "$src" "$dst"
fi
{ set +x; } &>/dev/null
}
###############################################################################
# DeepDive shell
stage shell/deepdive bin/
stage shell/deepdive-version util/
stage shell/deepdive-help util/
stage shell/deepdive-env util/
stage shell/deepdive-whereis util/
stage shell/find-deepdive-app util/
stage shell/parse-url.sh util/
stage shell/error util/
stage shell/warning util/
stage shell/usage util/
stage shell/escape4sh util/
stage compiler/jq2sh util/
stage shell/jq util/
stage shell/logging-with-ts util/
stage shell/deepdive_bash_completion.sh etc/
# DeepDive compiler
stage compiler/deepdive-compile util/
stage compiler/deepdive-check util/
stage compiler/app-has-been-compiled util/
stage compiler/deepdive-default.conf etc/
stage compiler/compile-config util/
stage compiler/compile-check util/
stage compiler/compile-code util/
stage compiler/compile-codegen util/
stage compiler/ddlog-wrapper.sh bin/ddlog
stage .build/submodule/compiler/ddlog/target/scala-2.11/ddlog-assembly-0.1-SNAPSHOT.jar lib/ddlog.jar
stage .build/submodule/compiler/hocon2json/hocon2json.sh util/hocon2json
stage .build/submodule/compiler/hocon2json/target/scala-2.10/hocon2json-assembly-*.jar util/hocon2json.jar
# DeepDive execution planner and runner
stage runner/deepdive-run util/
stage runner/deepdive-plan util/
stage runner/deepdive-do util/
stage runner/deepdive-redo util/
stage runner/deepdive-mark util/
stage runner/deepdive-done util/
stage runner/format_timestamp util/
stage runner/reset_timestamp util/
stage runner/restore_timestamp util/
stage runner/mark_done util/
stage runner/resolve-args-to-do.sh util/
stage runner/show_progress util/
stage runner/ps_descendants util/
stage runner/deepdive-compute util/
stage runner/load-compute-driver.sh util/
stage runner/compute-driver/local util/compute-driver/
stage runner/compute-driver/torque util/compute-driver/
stage runner/computers-default.conf util/
stage .build/submodule/runner/mkmimo/mkmimo util/
# DeepDive database operations and drivers
stage database/deepdive-query util/
stage database/deepdive-db util/
stage database/deepdive-initdb util/
stage database/deepdive-sql util/
stage database/deepdive-create util/
stage database/deepdive-relation util/
stage database/deepdive-load util/
stage database/deepdive-unload util/
stage database/deepdive-insert-into util/
stage database/deepdive-alter util/
stage database/deepdive-drop util/
stage database/load-db-driver.sh util/
stage database/db-driver/postgresql util/db-driver/
stage database/db-driver/greenplum util/db-driver/
stage database/db-driver/postgresql-xl util/db-driver/
stage database/db-driver/mysql util/db-driver/
stage database/db-driver/drill util/db-driver/
stage database/partition_id_range util/
stage database/tsj2fmt util/
stage database/pgtsv_to_json util/
stage database/tsv2tsj util/
stage database/tsv2tsj.pl util/
stage database/tsv2json util/
stage util/partition_integers util/
# DDlib
stage ddlib/ddlib lib/python/
stage ddlib/deepdive.py lib/python/
# DeepDive inference engine and supporting utilities
# copying shared libraries required by the dimmwitted sampler and generating a wrapper
PATH="$PWD"/extern/buildkit:"$PATH"
for cmd in dw; do
stage .build/submodule/inference/dimmwitted/$cmd util/sampler-$cmd.bin
install-shared-libraries-required-by "$STAGE_DIR"/lib/dw \
.build/submodule/inference/dimmwitted/$cmd
generate-wrapper-for-libdirs "$STAGE_DIR"/util/sampler-$cmd \
"$STAGE_DIR"/util/sampler-$cmd.bin \
"$STAGE_DIR"/lib/dw
done
stage inference/deepdive-model util/
# DeepDive utilities
stage util/draw_calibration_plot util/
stage util/calibration.py util/
stage util/calibration.plg util/
stage .build/submodule/util/mindbender/@prefix@/ mindbender/
stage util/mindbender-wrapper.sh bin/mindbender
# runtime dependencies after building them from source
stage extern/.build/bundled lib/