Merge pull request #243 from Shopify/rwstauner/site-local

Import html site generation files, enable site to build and serve locally, fix several broken things

.gitignore

@@ -1,3 +1,5 @@
+/build/*
+/.bundle
 data/*.json
 data/*.csv
 data/*.txt
@@ -7,3 +9,4 @@ continuous_reporting/single_iter_data/*
 **/temp.json
 .DS_Store
 bench_params.json
+/vendor

Gemfile

@@ -6,4 +6,4 @@ gem "kramdown" # used in reporting
 gem "slack-ruby-client"
 
 gem "minitest", "~>5.11.3"
-
+gem "webrick"

Gemfile.lock

@@ -27,6 +27,7 @@ GEM
       hashie
       websocket-driver
     victor (0.3.3)
+    webrick (1.8.1)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -40,6 +41,7 @@ DEPENDENCIES
   rake
   slack-ruby-client
   victor
+  webrick
 
 BUNDLED WITH
    2.2.30

README.md

@@ -27,6 +27,8 @@ For that reason, where we-the-authors provide official numbers they will usually
 
 ## How to Use this Repo
 
+### Benchmark data
+
 You can run `./basic_benchmark.rb` to clone appropriate other repositories (yjit, yjit-bench) and run the benchmarks. You can also specify one or more benchmark names on the command line to run only those benchmarks: `./basic_benchmark.rb activerecord`
 
 `basic_benchmark.rb` also accepts many other parameters, such as a `--skip-git-updates` parameter for runs after the first to not "git pull" its repos and rebuild Ruby.
@@ -39,6 +41,24 @@ Use `basic_report.rb --help` to get started. There are several different reports
 
 You can find older examples of data-gathering scripts using `git log -- runners` (to see files that used to be in the "runners" directory) and post-processing scripts in the "formatters" directory.
 
+
+### speed.yjit.org site
+
+After collecting some data with `basic_benchmark.rb` you can generate the html
+site (hosted at speed.yjit.org) with a simple command:
+
+`site/exe serve`
+
+This will move files from `./data/` into a `build` directory,
+generate all the html files, and start a web server where you can view the site at `localhost:8000`.
+
+Some of the reports are built using `lib/yjit-metrics` and have templates in
+`lib/yjit-metrics/report_templates`.
+The rest of the files that build the site are found beneath `site`.
+There are `erb` files to generate additional pages
+and the script that does all the file rendering in `site/_framework/`.
+
+
 ## TruffleRuby
 
 Our experience has been that the JVM (non-default) version gets better results than the native/SubstrateVM version, so if you are going to install a release we recommend the truffleruby+graalvm variant (with ruby-build) or truffleruby-graalvm (with ruby-install).

basic_benchmark.rb

@@ -229,7 +229,7 @@
 
     config_desc = "Comma-separated list of Ruby configurations to test" + "\n\t\t\tfrom: #{CONFIG_NAMES.join(", ")}\n\t\t\tdefault: #{DEFAULT_CONFIGS.join(",")}"
     opts.on("--configs=CONFIGS", config_desc) do |configs|
-        configs_to_test = configs.split(",").map(&:strip).uniq
+        configs_to_test = configs.split(",").map(&:strip).map { |s| s.gsub('PLATFORM', YJITMetrics::PLATFORM) }.uniq
         bad_configs = configs_to_test - CONFIG_NAMES
         raise "Requested test configuration(s) don't exist: #{bad_configs.inspect}!\n\nLegal configs include: #{CONFIG_NAMES.inspect}" unless bad_configs.empty?
         wrong_platform_configs = configs_to_test - THIS_PLATFORM_CONFIGS

continuous_reporting/generate_and_upload_reports.rb

@@ -11,11 +11,10 @@
 
 YM_REPO = YJITMetrics::ContinuousReporting::YM_REPO
 RAW_BENCHMARK_ROOT = YJITMetrics::ContinuousReporting::RAW_BENCHMARK_ROOT
-RAW_REPORTS_ROOT = YJITMetrics::ContinuousReporting::RAW_REPORTS_ROOT
 BUILT_REPORTS_ROOT = YJITMetrics::ContinuousReporting::BUILT_REPORTS_ROOT
 GHPAGES_REPO = YJITMetrics::ContinuousReporting::GHPAGES_REPO
 
-[RAW_BENCHMARK_ROOT, RAW_REPORTS_ROOT, BUILT_REPORTS_ROOT].each do |dir|
+[RAW_BENCHMARK_ROOT, BUILT_REPORTS_ROOT].each do |dir|
   unless File.exist?(dir)
     raise "We expected directory #{dir.inspect} to exist in order to generate reports!"
   end
@@ -52,7 +51,7 @@
     },
     "blog_exit_reports" => {
         report_type: :basic_report,
-        extensions: [ "bench_list.html" ], # Funny thing here - we generate a *lot* of exit report files, but rarely with a fixed name.
+        extensions: [ "bench_list.txt" ], # Funny thing here - we generate a *lot* of exit report files, but rarely with a fixed name.
     },
     "iteration_count" => {
         report_type: :basic_report,
@@ -167,7 +166,9 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl
 timestamps = json_timestamps.keys.sort
 timestamps.each do |ts|
     test_files = json_timestamps[ts]
+  if ENV['ALLOW_ARM_ONLY_REPORTS'] != '1'
     next unless test_files.any? { |tf| tf.include?("x86") } # Right now, ARM-only reports are very buggy.
+  end
     do_regenerate_year = regenerate_year && ts.start_with?(regenerate_year)
 
     REPORTS_AND_FILES.each do |report_name, details|
@@ -285,17 +286,8 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl
     # TODO: figure out a new way to verify that appropriate files were written. With various subdirs, the old way won't cut it.
 end
 
-# Switch to raw-yjit-reports, which symlinks to the built reports
-Dir.chdir(RAW_REPORTS_ROOT)
-puts "Switched to #{Dir.pwd}"
-YJITMetrics.check_call "git checkout main && git pull"
-
 # Make sure it builds locally
-# Funny thing here - this picks up the Bundler config from this script, via env vars.
-# So it's important to include the kramdown gem, and others used in reporting, in
-# the yjit-metrics Gemfile. Or you can run generate_and_upload_reports.rb from the
-# other directory, where it picks up the reporting Gemfile. That works too.
-YJITMetrics.check_call "bundle exec ruby -I./_framework _framework/render.rb build"
+YJITMetrics.check_call "site/exe build"
 
 puts "Static site seems to build correctly. That means that GHPages should do the right thing on push."
 
@@ -317,23 +309,26 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl
 # Copy built _site directory into raw pages repo as a new single commit, to branch new_pages
 Dir.chdir GHPAGES_REPO
 puts "Switched to #{Dir.pwd}"
-YJITMetrics.check_call "git checkout empty"
-YJITMetrics.check_call "git branch -D new_pages || echo ok" # If the local new_pages branch exists, delete it
-YJITMetrics.check_call "git checkout --orphan new_pages"
-YJITMetrics.check_call "git rm --cached -r .gitignore && rm -f .gitignore"
-YJITMetrics.check_call "mv #{RAW_REPORTS_ROOT}/_site/* ./"
-YJITMetrics.check_call "touch .nojekyll"
-YJITMetrics.check_call "rm Gemfile Gemfile.lock" # Why aren't these excluded during render?
-YJITMetrics.check_call "git add ."
-YJITMetrics.check_call "git commit -m 'Rebuilt site HTML'"
-
-unless no_push
+
+# Currently this will only be true on the server.
+if File.exist?(".git")
+  YJITMetrics.check_call "git checkout empty"
+  YJITMetrics.check_call "git branch -D new_pages || echo ok" # If the local new_pages branch exists, delete it
+  YJITMetrics.check_call "git checkout --orphan new_pages"
+  YJITMetrics.check_call "git rm --cached -r .gitignore && rm -f .gitignore"
+  YJITMetrics.check_call "mv #{YM_REPO}/site/_site/* ./"
+  YJITMetrics.check_call "touch .nojekyll"
+  YJITMetrics.check_call "git add ."
+  YJITMetrics.check_call "git commit -m 'Rebuilt site HTML'"
+
+  unless no_push
     # Reset the pages branch to the new built site
     YJITMetrics.check_call "git checkout pages && git reset --hard new_pages"
     YJITMetrics.check_call "git push -f origin pages"
     YJITMetrics.check_call "git branch -D new_pages || echo ok" # Sometimes this fails for no obvious reason
-end
+  end
 
-YJITMetrics.check_call "git checkout empty"
+  YJITMetrics.check_call "git checkout empty"
+end
 
-puts "Finished generate_and_upload_reports successfully in #{RAW_REPORTS_ROOT}!"
+puts "Finished generate_and_upload_reports successfully in #{YM_REPO}!"

continuous_reporting/gh_tasks/check_reporting.sh

@@ -9,6 +9,8 @@ chruby 3.0.2
 cd ~/ym/yjit-metrics
 git pull
 
+bundle
+
 # Copy benchmark raw data into destination repo
 #ruby continuous_reporting/file_benchmark_data_into_raw.rb -d continuous_reporting/data
 

continuous_reporting/gh_tasks/check_reporting_minimal.sh

@@ -11,6 +11,8 @@ chruby 3.0.2
 
 cd ~/ym/yjit-metrics
 
+bundle
+
 # Copy benchmark raw data into destination repo
 #ruby continuous_reporting/file_benchmark_data_into_raw.rb -d continuous_reporting/data
 

continuous_reporting/gh_tasks/commit_benchmark_data.sh

@@ -16,6 +16,8 @@ chruby 3.0.2
 # If there is uncommitted data after a benchmark run, get it into (all?) raw_benchmarks where it belongs
 cd ~/ym/yjit-metrics
 
+bundle
+
 # NOTE: This data dir is not configurable. If we run a smoke test on jenkins
 # into a different data dir we explicitly do not want to include that here.
 # In that case this will run, find no data, and do nothing (which is what we want).

continuous_reporting/gh_tasks/generate_bench_params.sh

@@ -10,6 +10,8 @@ chruby 3.0.2
 cd ~/ym/yjit-metrics
 git checkout main && git pull
 
+bundle
+
 # No timestamp given, default to right now
 ruby continuous_reporting/create_json_params_file.rb --full-rebuild=$FULL_REBUILD --bench-type="$BENCH_TYPE" --yjit-metrics-name=$YJIT_METRICS_NAME --yjit-metrics-repo=$YJIT_METRICS_REPO --yjit-bench-name=$YJIT_BENCH_NAME --yjit-bench-repo=$YJIT_BENCH_REPO --cruby-name=$CRUBY_NAME --cruby-repo=$CRUBY_REPO --benchmark-data-dir=$BENCH_DATA_DIR
 

continuous_reporting/gh_tasks/report_and_upload.sh

@@ -10,6 +10,8 @@ chruby 3.0.2
 
 cd ~/ym/yjit-metrics
 
+bundle
+
 # Copy benchmark raw data into destination repo, generate reports, commit changes to Git.
 ruby continuous_reporting/generate_and_upload_reports.rb
 

continuous_reporting/gh_tasks/report_full_rebuild.sh

@@ -10,6 +10,8 @@ chruby 3.0.2
 
 cd ~/ym/yjit-metrics
 
+bundle
+
 # Copy benchmark raw data into destination repo, generate reports, commit changes to Git.
 # The --regenerate-reports argument will regenerate ***all*** reports, which can take quite a
 # long time. It will also occasionally hit a Ruby error, so we should update from 3.0.2 when

continuous_reporting/gh_tasks/run_benchmarks.sh

@@ -6,6 +6,8 @@ set -e
 chruby 3.0.2
 cd ~/ym/yjit-metrics
 
+bundle
+
 ruby continuous_reporting/benchmark_and_update.rb --no-gh-issue --no-perf-tripwires --bench-params=$BENCH_PARAMS
 
 echo "Completed benchmarking successfully."

continuous_reporting/gh_tasks/run_single_iter_benchmarks.sh

@@ -12,6 +12,8 @@ chruby 3.0.2
 
 cd ~/ym/yjit-metrics
 
+bundle
+
 ruby continuous_reporting/benchmark_and_update.rb --benchmark-type smoketest --no-gh-issue --no-perf-tripwires --bench-params=$BENCH_PARAMS --data-dir=continuous_reporting/single_iter_data
 
 echo "Completed smoke-test benchmarking successfully."

continuous_reporting/jenkins/Jenkinsfile_benchmark_smoke_run

@@ -17,6 +17,7 @@ pipeline {
     }
 
     environment {
+        BUNDLE_DEPLOYMENT = 'true'
         SLACK_OAUTH_TOKEN = credentials('684cd699-feae-4ef1-8483-e71440a73fcd')
         NOTIFIER_ARGS = "--template smoke_test --channels \"${params.SLACK_FAILURE_NOTIFY}\" --properties \"RUBY=${params.RUBY_VERSION},YJIT_BENCH=${params.YJIT_BENCH_VERSION},YJIT_METRICS=${params.YJIT_METRICS_VERSION}\""
     }

continuous_reporting/jenkins/Jenkinsfile_configurable_run

@@ -98,6 +98,10 @@ pipeline {
         )
     }
 
+    environment {
+      BUNDLE_DEPLOYMENT = 'true'
+    }
+
     stages {
         stage('generate_params') {
             environment {
@@ -173,7 +177,6 @@ pipeline {
                   script {
                     ssh "cd ym/raw-benchmark-data && git pull"
                     ssh "YJIT_METRICS_REPO=${params.YJIT_METRICS_REPO} YJIT_METRICS_NAME=${params.YJIT_METRICS_VERSION} ym/yjit-metrics/continuous_reporting/gh_tasks/git_update_yjit_metrics_repo.sh"
-                    ssh "ym/yjit-metrics/continuous_reporting/gh_tasks/git_update_pages_repo.sh"
                     ssh "ym/yjit-metrics/continuous_reporting/gh_tasks/report_and_upload.sh"
 
                     // The .max() method is not allowed without granting explicit access in jenkins.

continuous_reporting/jenkins/Jenkinsfile_data_checkin_and_upload

@@ -29,6 +29,10 @@ pipeline {
         )
     }
 
+    environment {
+        BUNDLE_DEPLOYMENT = 'true'
+    }
+
     stages {
         // This stage commits and pushes the benchmark data from its directory, but doesn't get it there from yjit-metrics/continuous_reporting/data
         stage('data checkin') {

continuous_reporting/jenkins/Jenkinsfile_full_benchmark_run

@@ -26,6 +26,7 @@ pipeline {
     }
 
     environment {
+        BUNDLE_DEPLOYMENT = 'true'
         SLACK_OAUTH_TOKEN = credentials('684cd699-feae-4ef1-8483-e71440a73fcd')
         NOTIFIER_ARGS = "--template build_status --channels \"${params.SLACK_FAILURE_NOTIFY}\""
     }

continuous_reporting/jenkins/Jenkinsfile_reporting_rebuild

@@ -14,6 +14,7 @@ pipeline {
     agent any
 
     environment {
+        BUNDLE_DEPLOYMENT = 'true'
         SSH_HOST = credentials('2ea5f8cb-6fab-4454-b7cb-46ca2fd3cba7')
     }
 

lib/yjit-metrics.rb

@@ -29,11 +29,11 @@ module YJITMetrics
 
     HARNESS_PATH = File.expand_path(__dir__ + "/../metrics-harness")
 
-    PLATFORMS = ["x86_64", "arm", "aarch64"]
+    PLATFORMS = ["x86_64", "aarch64"]
 
-    uname_platform = `uname -m`.chomp.downcase
-    PLATFORM = PLATFORMS.detect { |platform| uname_platform.include?(platform) }
-    raise("yjit-metrics only supports running on x86_64 and arm64!") if !PLATFORM
+    uname_platform = `uname -m`.chomp.downcase.sub(/^arm(\d+)$/, 'aarch\1')
+    PLATFORM = PLATFORMS.detect { |platform| uname_platform == platform }
+    raise("yjit-metrics only supports running on x86_64 and aarch64!") if !PLATFORM
 
     # This structure is returned by the benchmarking harness from a run.
     JSON_RUN_FIELDS = %i(times warmups yjit_stats peak_mem_bytes failures_before_success benchmark_metadata ruby_metadata)
@@ -85,17 +85,17 @@ def chdir(dir, &block)
     # Checked system - error if the command fails
     def check_call(command)
         # Use prefix to makes it easier to see in the log.
-        puts("## [#{Time.now}] #{command}")
+        puts("\e[33m## [#{Time.now}] #{command}\e[00m")
 
         status = nil
         Benchmark.realtime do
           status = system(command)
         end.tap do |time|
-          printf "## (`#{command}` took %.2fs)\n", time
+          printf "\e[34m## (`#{command}` took %.2fs)\e[00m\n", time
         end
 
         unless status
-            puts "Command #{command.inspect} failed in directory #{Dir.pwd}"
+            puts "\e[31mCommand #{command.inspect} failed in directory #{Dir.pwd}\e[00m"
             raise RuntimeError.new
         end
     end

lib/yjit-metrics/bench-results.rb

@@ -32,6 +32,8 @@ def geomean_or_nil(values)
     end
 
     def stddev(values)
+        return 0 if values.size <= 1
+
         xbar = mean(values)
         diff_sqrs = values.map { |v| (v-xbar)*(v-xbar) }
         # Bessel's correction requires dividing by length - 1, not just length:
@@ -402,8 +404,11 @@ def add_for_config(config_name, benchmark_results, normalize_bench_names: true)
                 raise "No arch provided in data file, and no x86_64 detected in RUBY_DESCRIPTION!"
             end
         end
-        ruby_meta["platform"] ||= YJITMetrics::PLATFORMS.detect { |platform| (ruby_meta["uname -a"] || "").downcase.include?(platform) }
-        ruby_meta["platform"] ||= YJITMetrics::PLATFORMS.detect { |platform| (ruby_meta["arch"] || "").downcase.include?(platform) }
+        recognized_platforms = YJITMetrics::PLATFORMS + ["arm64"]
+        ruby_meta["platform"] ||= recognized_platforms.detect { |platform| (ruby_meta["uname -a"] || "").downcase.include?(platform) }
+        ruby_meta["platform"] ||= recognized_platforms.detect { |platform| (ruby_meta["arch"] || "").downcase.include?(platform) }
+        raise "Uknown platform" if !ruby_meta["platform"]
+        ruby_meta["platform"].sub!(/^arm(\d+)$/, 'aarch\1')
         #@platform ||= ruby_meta["platform"]
 
         #if @platform != ruby_meta["platform"]
@@ -786,4 +791,17 @@ def initialize(context)
         @context = context
     end
 
+    # Look for "PLATFORM_#{name}"; prefer specified platform if present.
+    def find_config(name, platform: "x86_64")
+      matches = @context[:configs].select { |c| c.end_with?(name) }
+      matches.detect { |c| c.start_with?(platform) } || matches.first
+    end
+
+    # Strip PLATFORM from beginning of name
+    def platform_of_config(config)
+      YJITMetrics::PLATFORMS.each do |p|
+        return p if config.start_with?("#{p}_")
+      end
+      raise "Unknown platform in config '#{config}'"
+    end
 end