From e79723bf2dd44e5c99c3833027d29f57567fcfa4 Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Thu, 19 Nov 2020 12:45:14 -0800 Subject: [PATCH] [toil] Debugging Debian locale issue with a dummy task. - Tried Ubuntu bionic image, but it doesn't have python2-dev :-/ - Try setting LANG. Doesn't seem to work. - benchmarks/time: Be pedantic about exit status --- .builds/dev-minimal.yml | 8 ++++++++ .builds/dummy.yml | 16 ++++++++++++++++ .travis.yml | 2 +- benchmarks/time-helper.c | 13 +++++++++---- benchmarks/time-test.sh | 38 ++++++++++++++++++++++++++++++++++++++ build/dev.sh | 35 ----------------------------------- services/sourcehut.sh | 10 ++++++++-- services/toil-worker.sh | 18 ++++++++++++------ 8 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 .builds/dummy.yml diff --git a/.builds/dev-minimal.yml b/.builds/dev-minimal.yml index e80e0aabe9..fa744c6958 100644 --- a/.builds/dev-minimal.yml +++ b/.builds/dev-minimal.yml @@ -1,5 +1,10 @@ image: debian/buster +# doesn't have python2-dev! +#image: ubuntu/18.04 + packages: + # for utf-8 + - locales # for toil - zip @@ -20,6 +25,9 @@ packages: - python3-setuptools - python3-pip +environment: + LANG: en_US.UTF-8 + secrets: - 2678474d-b22b-449f-a19a-16cb403c94cd diff --git a/.builds/dummy.yml b/.builds/dummy.yml new file mode 100644 index 0000000000..add1198ffe --- /dev/null +++ b/.builds/dummy.yml @@ -0,0 +1,16 @@ +image: debian/buster +packages: + # for utf-8 + - locales + - zip +environment: + LANG: en_US.UTF-8 +secrets: + - 2678474d-b22b-449f-a19a-16cb403c94cd +tasks: + - dummy: | + cd oil + services/toil-worker.sh run-dummy + - publish-html: | + cd oil + services/sourcehut.sh publish-html-assuming-ssh-key diff --git a/.travis.yml b/.travis.yml index 3d378f4603..a0695d04f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -120,8 +120,8 @@ jobs: - mksh - zsh # since we're not using language: python - - python3-setuptools - python-pip + - python3-setuptools - python3-pip install: diff --git a/benchmarks/time-helper.c b/benchmarks/time-helper.c index b06e32fc1c..101d8a7d06 100644 --- a/benchmarks/time-helper.c +++ b/benchmarks/time-helper.c @@ -90,15 +90,20 @@ int time_helper(Spec *spec, FILE* f) { struct timeval *user = &usage.ru_utime; struct timeval *sys = &usage.ru_stime; + int exit_status = -1; + if (WIFEXITED(status)) { + exit_status = WEXITSTATUS(status); + } + char d = spec->delimiter; // NO delimiter at first! - if (spec->x) { int_cell(f, 0, WEXITSTATUS(status)); } + if (spec->x) { int_cell(f, 0, exit_status); } if (spec->e) { time_cell(f, d, &elapsed); } if (spec->U) { time_cell(f, d, &usage.ru_utime); } if (spec->S) { time_cell(f, d, &usage.ru_stime); } if (spec->M) { int_cell(f, d, usage.ru_maxrss); } - return status; + return exit_status; } int main(int argc, char **argv) { @@ -159,8 +164,8 @@ int main(int argc, char **argv) { char* mode = spec.append ? "a" : "w"; FILE* f = fopen(spec.out_path, mode); - int status = time_helper(&spec, f); + int exit_status = time_helper(&spec, f); fclose(f); - return status; + return exit_status; } diff --git a/benchmarks/time-test.sh b/benchmarks/time-test.sh index e72a750417..4fed9be19c 100755 --- a/benchmarks/time-test.sh +++ b/benchmarks/time-test.sh @@ -194,6 +194,42 @@ test-maxrss() { cat _tmp/maxrss } +test-time-helper() { + set +o errexit + + local tmp=_tmp/time-helper.txt + + local th=_devbuild/bin/time-helper + + # Make some work show up + local cmd='{ md5sum */*.md; sleep 0.15; exit 42; } > /dev/null' + + echo 'will be overwritten' > $tmp + cat $tmp + + $th + assert $? -ne 0 # it's 1, but could be 2 + + $th /bad + assert $? -eq 1 + + $th -o $tmp -d $'\t' -x -e -- sh -c "$cmd" + assert $? -eq 42 + cat $tmp + echo + + # Now append + $th -o $tmp -a -d , -x -e -U -S -M -- sh -c "$cmd" + assert $? -eq 42 + cat $tmp + echo + + # Error case + $th -z + assert $? -eq 2 +} + + all-passing() { test-usage test-csv @@ -205,6 +241,8 @@ all-passing() { test-rusage test-maxrss + test-time-helper + echo echo "All tests in $0 passed." } diff --git a/build/dev.sh b/build/dev.sh index a426840f6b..1cc221a581 100755 --- a/build/dev.sh +++ b/build/dev.sh @@ -328,41 +328,6 @@ time-helper() { ls -l $out } -time-helper-test() { - set +o errexit - - local tmp=_tmp/time-helper.txt - - local th=_devbuild/bin/time-helper - - # Make some work show up - local cmd='{ md5sum */*.md; sleep 0.15; exit 42; } > /dev/null' - - echo 'will be overwritten' > $tmp - cat $tmp - - $th - echo status=$? - - $th /bad - echo status=$? - - $th -o $tmp -d $'\t' -x -e -- sh -c "$cmd" - echo status=$? - cat $tmp - echo - - # Now append - $th -o $tmp -a -d , -x -e -U -S -M -- sh -c "$cmd" - echo status=$? - cat $tmp - echo - - # Error case - $th -z - echo status=$? -} - # Prerequisites: build/codegen.sh {download,install}-re2c all() { rm -f *.so # 12/2019: to clear old symlinks, maybe get rid of diff --git a/services/sourcehut.sh b/services/sourcehut.sh index d89a1277e6..831580c20f 100755 --- a/services/sourcehut.sh +++ b/services/sourcehut.sh @@ -14,6 +14,7 @@ source services/travis.sh # # https://man.sr.ht/tutorials/getting-started-with-builds.md # https://man.sr.ht/builds.sr.ht/#secrets +# https://man.sr.ht/builds.sr.ht/compatibility.md # # Basically, it supports up to 4 files called .builds/*.yml. # And we need to upload an SSH key as secret via the web UI. @@ -78,8 +79,13 @@ publish-html-assuming-ssh-key() { remote-cleanup-jobs-index 'srht-' # toil-worker.sh recorded this for us - return $(cat _tmp/toil/exit-status.txt) -} + local status + status=$(cat _tmp/toil/exit-status.txt) + + log "Exiting with saved status $status" + #return $status + return 42 +} "$@" diff --git a/services/toil-worker.sh b/services/toil-worker.sh index 818b711074..369e4677f1 100755 --- a/services/toil-worker.sh +++ b/services/toil-worker.sh @@ -38,6 +38,12 @@ dump-versions() { python3 -V } +dump-locale() { + locale + + return 42 +} + dummy-tasks() { ### Print tasks that execute quickly @@ -45,13 +51,10 @@ dummy-tasks() { cat <