From c9aeab3957c9e07a13bfcfb3f46382a918dc3bb4 Mon Sep 17 00:00:00 2001
From: Andy Chu <andy@oilshell.org>
Date: Thu, 19 Nov 2020 11:44:47 -0800
Subject: [PATCH] [toil] Porting job index to sr.ht

We have pages showing up!  The data model is slightly different than
that of Travis.
---
 benchmarks/time-test.sh |  2 +-
 services/sourcehut.sh   |  6 +++
 services/toil-worker.sh | 27 +++++++++----
 services/toil_web.py    | 86 +++++++++++++++++++++++++++--------------
 services/travis.sh      |  4 ++
 5 files changed, 88 insertions(+), 37 deletions(-)

diff --git a/benchmarks/time-test.sh b/benchmarks/time-test.sh
index 254448712a..e72a750417 100755
--- a/benchmarks/time-test.sh
+++ b/benchmarks/time-test.sh
@@ -185,7 +185,7 @@ test-rusage() {
 
 # Compare vs. /usr/bin/time.
 test-maxrss() {
-  if command -v time; then  # Ignore this on continuous build
+  if which time; then  # Ignore this on continuous build
     command time --format '%x %U %M' -- seq 1
   fi
 
diff --git a/services/sourcehut.sh b/services/sourcehut.sh
index e319fab991..900774d477 100755
--- a/services/sourcehut.sh
+++ b/services/sourcehut.sh
@@ -39,7 +39,13 @@ deploy-job-results() {
   export TASK_RUN_START_TIME=$(cat _tmp/toil/task-run-start-time.txt)
   export TASK_DEPLOY_START_TIME=$(date +%s)
 
+  # Because we don't have TRAVIS_JOB_NAME, etc.
+  export TOIL_JOB_NAME=$(cat _tmp/toil-job-name.txt)
+
   services/env_to_json.py \
+    TASK_RUN_START_TIME \
+    TASK_DEPLOY_START_TIME \
+    TOIL_JOB_NAME \
     JOB_ID \
     JOB_URL \
     > $job_id.json
diff --git a/services/toil-worker.sh b/services/toil-worker.sh
index 3015c941e5..7a5e6ffc79 100755
--- a/services/toil-worker.sh
+++ b/services/toil-worker.sh
@@ -169,13 +169,11 @@ EOF
 
 run-tasks() {
   ### Run the tasks on stdin and write _tmp/toil/INDEX.tsv.
+  local out_dir=$1  # should already exist
 
   # So we can always run benchmarks/time_.py.  TODO: Use Ninja for deps.
   build/dev.sh time-helper
 
-  local out_dir=_tmp/toil
-  mkdir -p $out_dir
-
   # For the later deploy step to pick up
   date +%s > $out_dir/task-run-start-time.txt
 
@@ -270,17 +268,32 @@ _run-dev-all-nix() {
 
 }
 
-run-ovm-tarball() { ovm-tarball-tasks | run-tasks; }
+job-main() {
+  local job_name=$1
+
+  local out_dir=_tmp/toil
+  mkdir -p $out_dir
+  echo "$job_name" > $out_dir/job-name.txt
+
+  ${job_name}-tasks | run-tasks $out_dir
+}
+
+run-ovm-tarball() { job-main 'ovm-tarball'; }
 
-run-app-tests() { app-tests-tasks | run-tasks; }
+run-app-tests() { job-main 'app-tests'; }
 
-run-cpp() { cpp-tasks | run-tasks; }
+run-cpp() { job-main 'cpp'; }
 
-run-other-tests() { other-tests-tasks | run-tasks; }
+run-other-tests() { job-main 'other-tests'; }
 
 run-dev-all-nix() {
   ### Travis job dev-all-nix
 
+  local job_name='dev-all-nix'
+  local out_dir=_tmp/toil
+  mkdir -p $out_dir
+  echo "$job_name" > $out_dir/job-name.txt
+
   # Run tasks the nix environment
   nix-shell \
     --argstr dev "none" \
diff --git a/services/toil_web.py b/services/toil_web.py
index ecfd843ed2..c1a224c7c5 100755
--- a/services/toil_web.py
+++ b/services/toil_web.py
@@ -142,17 +142,24 @@ def ParseJobs(stdin):
 
     meta['start_time_str'] = start_time_str
 
+    # Metadata for "Build".  Travis has this concept, but sourcehut doesn't.
+    # A build consists of many jobs.
+    meta['git_branch'] = meta.get('TRAVIS_BRANCH') or '?'
+
     try:
       commit_line = meta['TRAVIS_COMMIT_MESSAGE'].splitlines()[0]
-    except AttributeError:
+    except KeyError:
       commit_line = '?'
     meta['commit_line'] = commit_line
 
-    try:
-      commit_hash = meta['TRAVIS_COMMIT'][-8:]  # last 8 chars
-    except TypeError:
-      commit_hash = '?'
-    meta['commit_hash']  = commit_hash
+    meta['commit_hash'] = meta.get('TRAVIS_COMMIT') or '?'
+    meta['commit_hash_short'] = meta['commit_hash'][-8:]  # last 8 chars
+
+    # Metadata for "Job"
+
+    meta['job_name'] = meta.get('TOIL_JOB_NAME') or '?'  # Also be TRAVIS_JOB_NAME
+    meta['job_num'] = meta.get('TRAVIS_JOB_NUMBER') or meta.get('JOB_ID') or '?'
+    meta['job_url'] = meta.get('TRAVIS_JOB_WEB_URL') or meta.get('JOB_URL') or '?'
 
     filename = os.path.basename(json_path)
     basename, _ = os.path.splitext(filename)
@@ -166,9 +173,9 @@ def ParseJobs(stdin):
 </tr>
 <tr class="commit-row">
   <td colspan=2>
-    <code>%(TRAVIS_BRANCH)s</code>
+    <code>%(git_branch)s</code>
     &nbsp;
-    <code><a href="https://github.com/oilshell/oil/commit/%(TRAVIS_COMMIT)s">%(commit_hash)s</a></code>
+    <code><a href="https://github.com/oilshell/oil/commit/%(commit_hash)s">%(commit_hash_short)s</a></code>
   </td>
   <td class="commit-line" colspan=3>
     <code>%(commit_line)s</code>
@@ -182,9 +189,9 @@ def ParseJobs(stdin):
 
 JOB_ROW_TEMPLATE = '''\
 <tr>
-  <td>%(TRAVIS_JOB_NUMBER)s</td>
-  <td> <code><a href="%(basename)s.wwz/">%(TRAVIS_JOB_NAME)s</a></code> </td>
-  <td><a href="%(TRAVIS_JOB_WEB_URL)s">%(start_time_str)s</a></td>
+  <td>%(job_num)s</td>
+  <td> <code><a href="%(basename)s.wwz/">%(job_name)s</a></code> </td>
+  <td><a href="%(job_url)s">%(start_time_str)s</a></td>
   <td>%(elapsed_str)s</td>
   <td>%(status_html)s</td>
   <!-- todo; spec details
@@ -217,6 +224,27 @@ def ParseJobs(stdin):
       </thead>
 '''
 
+INDEX_BOTTOM = '''\
+    </table>
+
+    <p>
+      <a href="raw.html">raw data</a>
+    </p>
+  </body>
+</html>
+'''
+
+# Sort by descending build number
+def ByBuildNum(row):
+  return int(row.get('TRAVIS_BUILD_NUMBER', 0))
+
+def ByTaskRunStartTime(row):
+  return int(row.get('TASK_RUN_START_TIME', 0))
+
+# These are ascending
+def BySourcehutJobId(row):
+  return int(row.get('JOB_ID', 0))
+
 
 def main(argv):
   action = argv[1]
@@ -230,8 +258,23 @@ def main(argv):
 
     print(INDEX_TOP)
 
-    rows = list(ParseJobs(sys.stdin))
-    #print(json.dump(rows, indent=2))
+    jobs  = list(ParseJobs(sys.stdin))
+
+    # sourcehut doesn't have this grouping.
+    #rows.sort(key=BySourcehutJobId, reverse=True)
+    #groups = itertools.groupby(rows, key=BySourcehutJobId)
+
+    # Sort by start time
+    jobs.sort(key=ByTaskRunStartTime, reverse=True)
+
+    # TODO: We don't have this info
+    # The first job should have the same branch/commit/commit_line
+    #print(BUILD_ROW_TEMPLATE % jobs[0])
+
+    for job in jobs:
+      print(JOB_ROW_TEMPLATE % job)
+
+    print(INDEX_BOTTOM)
 
   elif action == 'travis-index':
 
@@ -243,13 +286,6 @@ def main(argv):
     print(INDEX_TOP)
     rows = list(ParseJobs(sys.stdin))
 
-    # Sort by descending build number
-    def ByBuildNum(row):
-      return int(row.get('TRAVIS_BUILD_NUMBER', 0))
-
-    def ByTaskRunStartTime(row):
-      return int(row.get('TASK_RUN_START_TIME', 0))
-
     rows.sort(key=ByBuildNum, reverse=True)
     groups = itertools.groupby(rows, key=ByBuildNum)
     #print(list(groups))
@@ -269,15 +305,7 @@ def ByTaskRunStartTime(row):
       for job in jobs:
         print(JOB_ROW_TEMPLATE % job)
 
-    print('''\
-    </table>
-
-    <p>
-      <a href="raw.html">raw data</a>
-    </p>
-  </body>
-</html>
-  ''')
+    print(INDEX_BOTTOM)
 
   elif action == 'cleanup':
     prefixes = []
diff --git a/services/travis.sh b/services/travis.sh
index f1dc7a424e..0dbb4edf0e 100755
--- a/services/travis.sh
+++ b/services/travis.sh
@@ -314,9 +314,13 @@ deploy-job-results() {
   export TASK_RUN_START_TIME=$(cat _tmp/toil/task-run-start-time.txt)
   export TASK_DEPLOY_START_TIME=$(date +%s)
 
+  # Redundant with TRAVIS_JOB_NAME
+  export TOIL_JOB_NAME=$(cat _tmp/toil-job-name.txt)
+
   services/env_to_json.py \
     TASK_RUN_START_TIME \
     TASK_DEPLOY_START_TIME \
+    TOIL_JOB_NAME \
     TRAVIS_JOB_NAME \
     TRAVIS_OS_NAME \
     TRAVIS_TIMER_START_TIME \