[toil] Improve the appearance of the dashboard.

- re-arrange the table - bust browser cache with query params, etc.
oilshell · Mar 17, 2020 · 7af7ea3 · 7af7ea3
1 parent 2c1dc9d
commit 7af7ea3
Show file tree

Hide file tree

Showing 5 changed files with 148 additions and 93 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,14 @@
 # Parallel Travis jobs for Oil.  We delegate most logic to
 # services/toil-worker.sh.
 
+# https://github.com/travis-ci/travis-ci/issues/1147
+# Might want to turn this on.
+
+#branches:
+#  only:
+#    - master
+
+
 # Every job uses this deploy step.
 _publish_html: &publish_html
   deploy:

diff --git a/services/toil-worker.sh b/services/toil-worker.sh
@@ -90,6 +90,9 @@ run-tasks() {
   local out_dir=_tmp/toil
   mkdir -p $out_dir
 
+  # For the later deploy step to pick up
+  date +%s > $out_dir/task-run-start-time.txt
+
   # This data can go on the dashboard index
   local tsv=$out_dir/INDEX.tsv
   rm -f $tsv

diff --git a/services/toil_web.py b/services/toil_web.py
@@ -1,16 +1,15 @@
 #!/usr/bin/env python2
 """
-toil_index.py
+toil_web.py
 
 Each continuous build run is assigned an ID.  Then it will generate:
 
-- $ID.git-log 
 - $ID.json  # metadata
 - $ID.tsv   # benchmarks/time.py output?  success/failure for each task
 - $ID.wwz   # files
 
-This script should generate an index.html with links to all the logs.
-
+This script generates an index.html with a table of metadata and links to the
+logs.
 """
 from __future__ import print_function
 
@@ -57,14 +56,25 @@ def ParseJobs(stdin):
     meta['max_status'] = max_status
     meta['total_elapsed'] = total_elapsed
 
-    # Note: this isn't what I think it is
+    # Note: this isn't a Unix timestamp
     #microseconds = int(meta['TRAVIS_TIMER_START_TIME']) / 1e6
     #log('ts = %d', microseconds)
+
+    # TODO: We could show "X minutes ago" etc.
     d = datetime.datetime.now()
-    meta['start_time_str'] = d.strftime('TODO')
+    meta['start_time_str'] = meta.get('TASK_RUN_START_TIME', '?')
+
+    try:
+      commit_line = meta['TRAVIS_COMMIT_MESSAGE'].splitlines()[0]
+    except AttributeError:
+      commit_line = '?'
+    meta['commit_line'] = commit_line
 
-    meta['commit_line'] = meta['TRAVIS_COMMIT_MESSAGE'].splitlines()[0]
-    meta['commit_hash'] = meta['TRAVIS_COMMIT'][-8:]  # last 8 chars
+    try:
+      commit_hash = meta['TRAVIS_COMMIT'][-8:]  # last 8 chars
+    except TypeError:
+      commit_hash = '?'
+    meta['commit_hash']  = commit_hash
 
     filename = os.path.basename(json_path)
     basename, _ = os.path.splitext(filename)
@@ -74,21 +84,34 @@ def ParseJobs(stdin):
 
 # TODO:
 # - Use JSON Template to escape it
-# - Red/Green for pass/fail from spec tests
+# - Red/Green for pass/fail (spec test CSS)
+# - Can we publish spec test numbers in JSON?
 
-ROW_TEMPLATE = '''
-<tr class="commit-row" style="padding-top: 1em">
+BUILD_ROW_TEMPLATE = '''\
+<tr class="spacer">
+  <td colspan=5><td/>
+</tr>
+<tr class="commit-row">
+  <td> %(TRAVIS_BUILD_NUMBER)s </td>
   <td> <code>%(TRAVIS_BRANCH)s</code> </td>
-  <td> <code>%(commit_hash)s</code> </td>
-  <td colspan="4">%(commit_line)s</td>
+  <td>
+    <code><a href="https://github.com/oilshell/oil/commit/%(TRAVIS_COMMIT)s">%(commit_hash)s</a></code>
+  </td>
+  <td class="commit-line" colspan="2">%(commit_line)s</td>
+</tr>
+<tr class="spacer">
+  <td colspan=5><td/>
 </tr>
+'''
+
+
+JOB_ROW_TEMPLATE = '''\
 <tr>
-  <td>%(max_status)d</td>
-  <td>%(start_time_str)s</td>
-  <td>%(total_elapsed).2f</td>
-  <td>%(TRAVIS_JOB_NAME)s</td>
   <td><a href="%(TRAVIS_JOB_WEB_URL)s">%(TRAVIS_JOB_NUMBER)s</a></td>
-  <td><a href="%(basename)s.wwz/">Details</a></td>
+  <td> <code>%(TRAVIS_JOB_NAME)s</code> </td>
+  <td>%(start_time_str)s</td>
+  <td><a href="%(basename)s.wwz/">%(total_elapsed).2f</a></td>
+  <td>%(max_status)d</td>
 </tr>
 '''
 
@@ -98,8 +121,10 @@ def main(argv):
 
   if action == 'index':
 
+    # Bust cache (e.g. Safari iPad seems to cache aggressively and doesn't
+    # have Ctrl-F5)
     html_head.Write(sys.stdout, 'Recent Jobs',
-        css_urls=['../web/base.css', '../web/toil.css'])
+        css_urls=['../web/base.css?cache=0', '../web/toil.css?cache=0'])
 
     print('''
   <body class="width40">
@@ -110,44 +135,66 @@ def main(argv):
 
     <h1>Recent Jobs</h1>
 
-    <p>
-      <a href="raw.html">raw data</a>
-    </p>
-
     <table>
       <thead>
+        <!--
+        <tr class="commit-row">
+          <td>Build #</td>
+          <td>Branch</td>
+          <td>Commit</td>
+          <td class="commit-line" colspan=2>Description</td>
+        </tr>
+        -->
         <tr>
-          <td>Status</td>
+          <td>Job #</td>
+          <td>Job Name</td>
           <td>Start Time</td>
           <td>Elapsed</td>
-          <td>Job Name</td>
-          <td>Job ID</td>
-          <td>Details</td>
+          <td>Status</td>
         </tr>
       </thead>
 ''')
 
-    # TODO:
-    # - Group by git commit
-    # - Show description and escape it
-
     rows = list(ParseJobs(sys.stdin))
-    rows.sort(
-        key=lambda row: int(row['TRAVIS_TIMER_START_TIME']), reverse=True)
+    import itertools
+
+    # Sort by descending build number
+    def ByBuildNum(row):
+      return int(row.get('TRAVIS_BUILD_NUMBER', 0))
+
+    def ByTaskRunStartTime(row):
+      return int(row.get('TASK_RUN_START_TIME', 0))
+
+    rows.sort(key=ByBuildNum, reverse=True)
+    groups = itertools.groupby(rows, key=ByBuildNum)
+    #print(list(groups))
+
+    for build_num, group in groups:
+      build_num = int(build_num)
+      log('---')
+      log('build %d', build_num)
 
-    for row in rows:
-      print(ROW_TEMPLATE % row)
+      jobs = list(group)
+
+      # Sort by start time
+      jobs.sort(key=ByTaskRunStartTime, reverse=True)
+
+      # The first job should have the same branch/commit/commit_line
+      print(BUILD_ROW_TEMPLATE % jobs[0])
+
+      for job in jobs:
+        print(JOB_ROW_TEMPLATE % job)
 
     print('''\
     </table>
+
+    <p>
+      <a href="raw.html">raw data</a>
+    </p>
   </body>
 </html>
   ''')
 
-    # TODO: read jobs on stdin
-    # - open .tsv and JSON
-    # - write HTML to output
-
   else:
     raise RuntimeError('Invalid action %r' % action)
 

diff --git a/services/travis.sh b/services/travis.sh
@@ -41,8 +41,8 @@ travis-html-head() {
 
   local base_url='../../web'
 
-  # These files live at the root
-  html-head --title "$title" "/web/base.css" "/web/toil.css" 
+  # These files live at the root.  Bust cache.
+  html-head --title "$title" "/web/base.css?cache=0" "/web/toil.css?cache=0" 
 }
 
 #
@@ -91,15 +91,6 @@ deploy-public-key() {
 readonly USER='travis_admin'
 readonly HOST='travis-ci.oilshell.org'
 
-# Print the list of Travis jobs.
-job-config() {
-  # (artifact, platform)
-  cat <<EOF
-dev-minimal ubuntu-xenial
-jobs        foo
-EOF
-}
-
 home-page() {
   ### travis-ci.oilshell.org home page
 
@@ -118,23 +109,15 @@ home-page() {
        <a href="https://github.com/oilshell/oil/wiki/Travis-CI-for-Oil">Travis CI for Oil</a> for details.
     </p>
 
-    <table>
-      <thead>
-        <tr>
-          <td>Artifact</td>
-          <td>Platform</td>
-        </tr>
-      </thead>
-EOF
-  job-config | while read artifact platform; do
-    echo "<tr>"
-    echo "  <td><a href="$artifact/">$artifact</a></td>"
-    echo "  <td>$platform</td>"
-    echo "</tr>"
-    echo
-  done
-  cat <<EOF
-    </table>
+    <ul>
+      <li>
+        <a href="jobs/">Jobs</a>
+      </li>
+      <li>
+        <a href="builds/">Builds</a>
+      </li>
+    </ul>
+
   </body>
 </html>
 EOF
@@ -314,7 +297,17 @@ deploy-job-results() {
 
   make-job-wwz $job_id
 
+  # Written by toil-worker.sh
+  # TODO:
+  # - Don't export these, just pass to env_to_json
+  # - if it exists, publish _tmp/spec/*.stats.txt and publish it?
+  #   - osh failures and total failures
+  export TASK_RUN_START_TIME=$(cat _tmp/toil/task-run-start-time.txt)
+  export TASK_DEPLOY_START_TIME=$(date +%s)
+
   services/env_to_json.py \
+    TASK_RUN_START_TIME \
+    TASK_DEPLOY_START_TIME \
     TRAVIS_JOB_NAME \
     TRAVIS_OS_NAME \
     TRAVIS_TIMER_START_TIME \
@@ -337,14 +330,10 @@ deploy-job-results() {
   log "http://travis-ci.oilshell.org/jobs/"
   log "http://travis-ci.oilshell.org/jobs/$job_id.wwz/"
   log ''
-
-  # TODO: git-log.txt, .json for hostname
-  # - $job_id.git-log.txt: commit, branch, commit date, author?
-  # - $job_id.json: hostname, date, etc.?
 }
 
 format-jobs-index() {
-  travis-html-head 'dev-minimal jobs'
+  travis-html-head 'Recent Jobs (raw data)'
 
   cat <<EOF
   <body class="width40">
@@ -353,12 +342,12 @@ format-jobs-index() {
       | <a href="//oilshell.org/">oilshell.org</a>
     </p>
 
-    <h1>Continuous Build: <code>dev-minimal</code> Jobs</h1>
+    <h1>Recent Jobs (raw data)</h1>
 
     <table>
       <thead>
         <tr>
-          <td>Task</td>
+          <td>Job Archive</td>
           <td>JSON</td>
           <td>TSV</td>
         </tr>
@@ -390,8 +379,6 @@ EOF
 write-jobs-raw() {
   ### Rewrite travis-ci.oilshell.org/jobs/raw.html
 
-  # TODO: replace with toil_web.py?
-
   log "Listing remote .wwz"
   list-remote-results > _tmp/listing.txt
   ls -l _tmp/listing.txt
@@ -426,20 +413,4 @@ publish-html() {
   remote-rewrite-jobs-index
 }
 
-# TODO:
-#
-# - job index
-#   - print JSON fields: job URL, etc.
-#   - SUM up the times
-#   - SUM up the failures -- did it fail?
-#     - do this in Python
-#   - show commit description and diffstats
-#     - you can embed this in the .wwz file
-#
-# Later:
-# - spec test HTML
-#
-# Nice to have:
-# - link to fast/compact git diff?  I don't like Githubs
-
 "$@"
diff --git a/web/toil.css b/web/toil.css
@@ -8,7 +8,33 @@ table {
 }
 
 thead {
-  font-weight: bold;
   background-color: #eee;
+  font-weight: bold;
+  font-size: medium;
+}
+
+td {
+  padding-top: 0.1em;
+  padding-bottom: 0.1em;
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+  text-align: right;
+}
+
+thead tr td {
+  padding-top: 1em;
+  padding-bottom: 1em;
+}
+
+.commit-line {
+  text-align: left;
+}
+
+.commit-row td {
+  padding-top: 0.5em;
+  padding-bottom: 0.5em;
+
+  background-color: oldlace;
+
 }