[toil] Debugging Debian locale issue with a dummy task.

- Tried Ubuntu bionic image, but it doesn't have python2-dev :-/
- Try setting LANG.  Doesn't seem to work.
- benchmarks/time: Be pedantic about exit status

.builds/dev-minimal.yml

@@ -1,5 +1,10 @@
 image: debian/buster
+# doesn't have python2-dev!
+#image: ubuntu/18.04
+
 packages:
+  # for utf-8
+  - locales
   # for toil
   - zip
 
@@ -20,6 +25,9 @@ packages:
   - python3-setuptools
   - python3-pip
 
+environment:
+  LANG: en_US.UTF-8
+
 secrets:
   - 2678474d-b22b-449f-a19a-16cb403c94cd
 

.builds/dummy.yml

@@ -0,0 +1,16 @@
+image: debian/buster
+packages:
+  # for utf-8
+  - locales
+  - zip
+environment:
+  LANG: en_US.UTF-8
+secrets:
+  - 2678474d-b22b-449f-a19a-16cb403c94cd
+tasks:
+  - dummy: |
+      cd oil
+      services/toil-worker.sh run-dummy
+  - publish-html: |
+      cd oil
+      services/sourcehut.sh publish-html-assuming-ssh-key

.travis.yml

@@ -120,8 +120,8 @@ jobs:
             - mksh
             - zsh
               # since we're not using language: python
-            - python3-setuptools
             - python-pip
+            - python3-setuptools
             - python3-pip
 
       install:

benchmarks/time-helper.c

@@ -90,15 +90,20 @@ int time_helper(Spec *spec, FILE* f) {
   struct timeval *user = &usage.ru_utime;
   struct timeval *sys  = &usage.ru_stime;
 
+  int exit_status = -1;
+  if (WIFEXITED(status)) {
+    exit_status = WEXITSTATUS(status);
+  }
+
   char d = spec->delimiter;
   // NO delimiter at first!
-  if (spec->x) { int_cell(f, 0, WEXITSTATUS(status)); }
+  if (spec->x) { int_cell(f, 0, exit_status); }
   if (spec->e) { time_cell(f, d, &elapsed); }
   if (spec->U) { time_cell(f, d, &usage.ru_utime); }
   if (spec->S) { time_cell(f, d, &usage.ru_stime); }
   if (spec->M) { int_cell(f, d, usage.ru_maxrss); }
 
-  return status;
+  return exit_status;
 }
 
 int main(int argc, char **argv) {
@@ -159,8 +164,8 @@ int main(int argc, char **argv) {
 
   char* mode = spec.append ? "a" : "w";
   FILE* f = fopen(spec.out_path, mode);
-  int status = time_helper(&spec, f);
+  int exit_status = time_helper(&spec, f);
   fclose(f);
 
-  return status;
+  return exit_status;
 }

benchmarks/time-test.sh

@@ -194,6 +194,42 @@ test-maxrss() {
   cat _tmp/maxrss
 }
 
+test-time-helper() {
+  set +o errexit
+
+  local tmp=_tmp/time-helper.txt
+
+  local th=_devbuild/bin/time-helper
+
+  # Make some work show up
+  local cmd='{ md5sum */*.md; sleep 0.15; exit 42; } > /dev/null'
+
+  echo 'will be overwritten' > $tmp
+  cat $tmp
+
+  $th
+  assert $? -ne 0  # it's 1, but could be 2
+
+  $th /bad
+  assert $? -eq 1
+
+  $th -o $tmp -d $'\t' -x -e -- sh -c "$cmd"
+  assert $? -eq 42
+  cat $tmp
+  echo
+
+  # Now append
+  $th -o $tmp -a -d , -x -e -U -S -M -- sh -c "$cmd"
+  assert $? -eq 42
+  cat $tmp
+  echo
+
+  # Error case
+  $th -z
+  assert $? -eq 2
+}
+
+
 all-passing() {
   test-usage
   test-csv
@@ -205,6 +241,8 @@ all-passing() {
   test-rusage
   test-maxrss
 
+  test-time-helper
+
   echo
   echo "All tests in $0 passed."
 }

build/dev.sh

@@ -328,41 +328,6 @@ time-helper() {
   ls -l $out
 }
 
-time-helper-test() {
-  set +o errexit
-
-  local tmp=_tmp/time-helper.txt
-
-  local th=_devbuild/bin/time-helper
-
-  # Make some work show up
-  local cmd='{ md5sum */*.md; sleep 0.15; exit 42; } > /dev/null'
-
-  echo 'will be overwritten' > $tmp
-  cat $tmp
-
-  $th
-  echo status=$?
-
-  $th /bad
-  echo status=$?
-
-  $th -o $tmp -d $'\t' -x -e -- sh -c "$cmd"
-  echo status=$?
-  cat $tmp
-  echo
-
-  # Now append
-  $th -o $tmp -a -d , -x -e -U -S -M -- sh -c "$cmd"
-  echo status=$?
-  cat $tmp
-  echo
-
-  # Error case
-  $th -z
-  echo status=$?
-}
-
 # Prerequisites: build/codegen.sh {download,install}-re2c
 all() {
   rm -f *.so  # 12/2019: to clear old symlinks, maybe get rid of

services/sourcehut.sh

@@ -14,6 +14,7 @@ source services/travis.sh
 #
 # https://man.sr.ht/tutorials/getting-started-with-builds.md
 # https://man.sr.ht/builds.sr.ht/#secrets
+# https://man.sr.ht/builds.sr.ht/compatibility.md
 #
 # Basically, it supports up to 4 files called .builds/*.yml.
 # And we need to upload an SSH key as secret via the web UI.
@@ -78,8 +79,13 @@ publish-html-assuming-ssh-key() {
   remote-cleanup-jobs-index 'srht-'
 
   # toil-worker.sh recorded this for us
-  return $(cat _tmp/toil/exit-status.txt)
-}
+  local status
+  status=$(cat _tmp/toil/exit-status.txt)
+
+  log "Exiting with saved status $status"
 
+  #return $status
+  return 42
+}
 
 "$@"

services/toil-worker.sh

@@ -38,20 +38,23 @@ dump-versions() {
   python3 -V
 }
 
+dump-locale() {
+  locale
+
+  return 42
+}
+
 dummy-tasks() {
   ### Print tasks that execute quickly
 
   # (task_name, script, action, result_html)
   cat <<EOF
 dump-env      services/toil-worker.sh dump-env      -
 dump-timezone services/toil-worker.sh dump-timezone -
+dump-locale   services/toil-worker.sh dump-locale   -
 EOF
 }
 
-run-dummy() {
-  dummy-tasks | run-tasks
-}
-
 dev-minimal-tasks() {
   ### Print tasks for the 'dev-minimal' build
 
@@ -160,7 +163,6 @@ other-tests-tasks() {
   # repo overview is suggested by README.md
   cat <<EOF
 time-test         benchmarks/time-test.sh all-passing     -
-time-helper-test  build/dev.sh time-helper-test           -
 csv-concat-test   devtools/csv-concat-test.sh travis      -
 repo-overview     metrics/source-code.sh travis           -
 osh2oil           test/osh2oil.sh all-passing             -
@@ -181,6 +183,7 @@ run-tasks() {
   local tsv=$out_dir/INDEX.tsv
   rm -f $tsv
 
+  local status
   local max_status=0
 
   while read task_name script action result_html; do
@@ -196,7 +199,7 @@ run-tasks() {
     status=$?
     set -o errexit
 
-    if test $status -gt $max_status; then
+    if test "$status" -gt "$max_status"; then
       max_status=$status
     fi
 
@@ -206,6 +209,7 @@ run-tasks() {
     echo $'status\telapsed\ttask\tscript\taction\tresult_html'
     tail -n 1 $tsv
     echo
+    log "status=$status max_status=$max_status"
   done
 
   log '--- done ---'
@@ -271,6 +275,8 @@ job-main() {
   ${job_name}-tasks | run-tasks $out_dir
 }
 
+run-dummy() { job-main 'dummy'; }
+
 run-dev-minimal() { job-main 'dev-minimal'; }
 
 run-other-tests() { job-main 'other-tests'; }