diff --git a/bin/test.sh b/bin/test.sh index 124c1605..a222f6aa 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -100,7 +100,7 @@ EOF cat _tmp/reports.csv _tmp/bad_rows.txt > _tmp/reports_bad_rows.csv # Define a string variable and a boolean varaible. - cat >_tmp/schema.csv <_tmp/rappor-vars.csv < + +# You can source this file and use the alarm-status function. + +set -o nounset +set -o pipefail +set -o errexit + +# Run a command with a timeout, and print its status to a directory. +# +# Usage: +# alarm-status job_dir/STATUS 10 \ +# flaky_command ... + +alarm-status() { + set +o errexit + local status_file=$1 + shift # everything except the status file goes to perl + + # NOTE: It would be nice to setpgrp() before exec? And then can the signal + # be delivered to the entire group, like kill -SIGALRM -PID? + + # NOTE: If we did this in Python, the error message would also be clearer. + perl -e 'alarm shift; exec @ARGV or die "ERROR: after exec @ARGV"' "$@" + local exit_code=$? + + set -o errexit + + local result='' + case $exit_code in + 0) + # Would be nice to show elapsed time? + result='OK' + ;; + 9) + # decode_assoc.R will exit 9 if there are no reports AFTER + # --remove-bad-rows. A task can also be marked SKIPPED before running + # the child process (see backfill.sh). + result='SKIPPED by child process' + ;; + # exit code 142 means SIGALARM. 128 + 14 = 142. See 'kill -l'. + 142) + local seconds=$1 + result="TIMEOUT after $seconds seconds" + ;; + *) + result="FAIL with status $exit_code" + ;; + esac + echo "$result" + echo "$result" > $status_file +} + +_work() { + local n=10 # 2 seconds + for i in $(seq $n); do + echo $i - "$@" + sleep 0.2 + done +} + +_succeed() { + _work "$@" + exit 0 +} + +_fail() { + _work "$@" + exit 1 +} + +_skip() { + exit 9 +} + +# http://perldoc.perl.org/functions/alarm.html +# +# Delivers alarm. But how to get the process to have a distinct exit code? + +demo() { + mkdir -p _tmp + + # timeout + alarm-status _tmp/A 1 $0 _succeed foo + echo + + # ok + alarm-status _tmp/B 3 $0 _succeed bar + echo + + # fail + alarm-status _tmp/C 3 $0 _fail baz + echo + + # skip + alarm-status _tmp/D 3 $0 _skip baz + echo + + head _tmp/{A,B,C,D} +} + +test-simple() { + alarm-status _tmp/status.txt 1 sleep 2 +} + +test-bad-command() { + alarm-status _tmp/status.txt 1 nonexistent_sleep 2 +} + +# BUG +test-perl() { + set +o errexit + perl -e 'alarm shift; exec @ARGV or die "ERROR after exec @ARGV"' 1 _sleep 2 + echo $? +} + +if test $(basename $0) = 'alarm-lib.sh'; then + "$@" +fi diff --git a/pipeline/assoc.sh b/pipeline/assoc.sh new file mode 100755 index 00000000..38a16c37 --- /dev/null +++ b/pipeline/assoc.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Usage: +# ./assoc.sh + +set -o nounset +set -o pipefail +set -o errexit + +readonly THIS_DIR=$(dirname $0) +readonly RAPPOR_SRC=$(cd $THIS_DIR/.. && pwd) + +source $RAPPOR_SRC/util.sh # log, banner +source $RAPPOR_SRC/pipeline/tools-lib.sh +source $RAPPOR_SRC/pipeline/alarm-lib.sh + +# Run a single decode-assoc process, to analyze one variable pair for one +# metric. The arguments to this function are one row of the task spec. +decode-one() { + # Job constants, from decode-many + local rappor_src=$1 + local timeout_secs=$2 + local min_reports=$3 + local job_dir=$4 + local sample_size=$5 + + # Task spec variables, from task_spec.py + local num_reports=$6 + local metric_name=$7 + local date=$8 # for output naming only + local reports=$9 # file with reports + local var1=${10} + local var2=${11} + local map1=${12} + local output_dir=${13} + + local log_file=$output_dir/assoc-log.txt + local status_file=$output_dir/assoc-status.txt + mkdir --verbose -p $output_dir + + # Flags drived from job constants + local schema=$job_dir/config/rappor-vars.csv + local params_dir=$job_dir/config + local em_executable=$rappor_src/analysis/cpp/_tmp/fast_em + + # TODO: + # - Skip jobs with few reports, like ./backfill.sh analyze-one. + + # Output the spec for combine_status.py. + echo "$@" > $output_dir/assoc-spec.txt + + # NOTE: Not passing --num-cores since we're parallelizing already. + + # NOTE: --tmp-dir is the output dir. Then we just delete all the .bin files + # afterward so we don't copy them to x20 (they are big). + + { time \ + alarm-status $status_file $timeout_secs \ + $rappor_src/bin/decode-assoc \ + --create-bool-map \ + --remove-bad-rows \ + --em-executable $em_executable \ + --schema $schema \ + --params-dir $params_dir \ + --metric-name $metric_name \ + --reports $reports \ + --var1 $var1 \ + --var2 $var2 \ + --map1 $map1 \ + --reports-sample-size $sample_size \ + --tmp-dir $output_dir \ + --output-dir $output_dir + } >$log_file 2>&1 +} + +test-decode-one() { + decode-one $RAPPOR_SRC +} + +readonly DEFAULT_MIN_REPORTS=5000 + +#readonly DEFAULT_TIMEOUT_SECONDS=300 # 5 minutes as a quick test. +readonly DEFAULT_TIMEOUT_SECONDS=3600 # 1 hour + +readonly DEFAULT_MAX_PROCS=6 # TODO: Share with backfill.sh + +# Limit to 1M for now. Raise it when we have a full run. +readonly DEFAULT_SAMPLE_SIZE=1000000 + +readonly NUM_ARGS=8 # number of tokens in the task spec, used for xargs + +# Run many decode-assoc processes in parallel. +decode-many() { + local job_dir=$1 + local spec_list=$2 + + # These 3 params affect speed + local timeout_secs=${3:-$DEFAULT_TIMEOUT_SECONDS} + local sample_size=${4:-$DEFAULT_SAMPLE_SIZE} + local max_procs=${5:-$DEFAULT_MAX_PROCS} + + local rappor_src=${6:-$RAPPOR_SRC} + local min_reports=${7:-$DEFAULT_MIN_REPORTS} + + time cat $spec_list \ + | xargs --verbose -n $NUM_ARGS -P $max_procs --no-run-if-empty -- \ + $0 decode-one $rappor_src $timeout_secs $min_reports $job_dir $sample_size +} + +# Combine assoc results and render HTML. + +combine-and-render-html() { + local jobs_base_dir=$1 + local job_dir=$2 + + banner "Combining assoc task status" + TOOLS-cook combine-assoc-task-status $jobs_base_dir $job_dir + + banner "Combining assoc results" + TOOLS-cook combine-assoc-results $jobs_base_dir $job_dir + + banner "Splitting out status per metric, and writing overview" + TOOLS-cook assoc-metric-status $job_dir + + TOOLS-gen-ui symlink-static assoc $job_dir + + banner "Building overview .part.html from CSV" + TOOLS-gen-ui assoc-overview-part-html $job_dir + + banner "Building metric .part.html from CSV" + TOOLS-gen-ui assoc-metric-part-html $job_dir + + banner "Building pair .part.html from CSV" + TOOLS-gen-ui assoc-pair-part-html $job_dir + + banner "Building day .part.html from CSV" + TOOLS-gen-ui assoc-day-part-html $job_dir +} + +# Temp files left over by the fast_em R <-> C++. +list-and-remove-bin() { + local job_dir=$1 + # If everything failed, we might not have anything to list/delete. + find $job_dir -name \*.bin | xargs --no-run-if-empty -- ls -l --si + find $job_dir -name \*.bin | xargs --no-run-if-empty -- rm -f --verbose +} + +"$@" diff --git a/pipeline/combine_results.py b/pipeline/combine_results.py new file mode 100755 index 00000000..be6f987c --- /dev/null +++ b/pipeline/combine_results.py @@ -0,0 +1,138 @@ +#!/usr/bin/python +"""Combines results from multiple days of a single metric. + +Feed it the STATUS.txt files on stdin. It then finds the corresponding +results.csv, and takes the top N items. + +Example: + +Date, "google.com,", yahoo.com +2015-03-01, 0.0, 0.9 +2015-03-02, 0.1, 0.8 + +Dygraphs can load this CSV file directly. + +TODO: Use different dygraph API? + +Also we need error bars. + + new Dygraph(document.getElementById("graphdiv2"), + [ + [1,10,100], + [2,20,80], + [3,50,60], + [4,70,80] + ], + { + labels: [ "Date", "failure", "timeout", "google.com" ] + }); +""" + +import collections +import csv +import json +import os +import sys + +import util + + +def CombineDistResults(stdin, c_out, num_top): + dates = [] + var_cols = collections.defaultdict(dict) # {name: {date: value}} + + seen_dates = set() + + for line in stdin: + status_path = line.strip() + + # Assume it looks like .../2015-03-01/STATUS.txt + task_dir = os.path.dirname(status_path) + date = os.path.basename(task_dir) + + # Get rid of duplicate dates. These could be caused by retries. + if date in seen_dates: + continue + + seen_dates.add(date) + + with open(status_path) as f: + status = f.readline().split()[0] # OK, FAIL, TIMEOUT, SKIPPED + + dates.append(date) + + if status != 'OK': + continue # won't have results.csv + + results_path = os.path.join(task_dir, 'results.csv') + with open(results_path) as f: + c = csv.reader(f) + unused_header = c.next() # header row + + # they are sorted by decreasing "estimate", which is what we want + for i in xrange(0, num_top): + try: + row = c.next() + except StopIteration: + # It's OK if it doesn't have enough + util.log('Stopping early') + break + + string, _, _, proportion, _, prop_low, prop_high = row + + # dygraphs has a weird format with semicolons: + # value;lower;upper,value;lower;upper. + + # http://dygraphs.com/data.html#csv + + # Arbitrarily use 4 digits after decimal point (for dygraphs, not + # directly displayed) + dygraph_triple = '%.4f;%.4f;%.4f' % ( + float(prop_low), float(proportion), float(prop_high)) + + var_cols[string][date] = dygraph_triple + + # Now print CSV on stdout. + cols = sorted(var_cols.keys()) # sort columns alphabetically + c_out.writerow(['date'] + cols) + + dates.sort() + + for date in dates: + row = [date] + for col in cols: + cell = var_cols[col].get(date) # None mean sthere is no row + row.append(cell) + c_out.writerow(row) + + #util.log("Number of dynamic cols: %d", len(var_cols)) + + +def CombineAssocResults(stdin, c_out, num_top): + header = ('dummy',) + c_out.writerow(header) + + +def main(argv): + action = argv[1] + + if action == 'dist': + num_top = int(argv[2]) # number of values to keep + c_out = csv.writer(sys.stdout) + CombineDistResults(sys.stdin, c_out, num_top) + + elif action == 'assoc': + num_top = int(argv[2]) # number of values to keep + c_out = csv.writer(sys.stdout) + CombineAssocResults(sys.stdin, c_out, num_top) + + else: + raise RuntimeError('Invalid action %r' % action) + + +if __name__ == '__main__': + try: + main(sys.argv) + except RuntimeError, e: + print >>sys.stderr, 'FATAL: %s' % e + sys.exit(1) diff --git a/pipeline/combine_results_test.py b/pipeline/combine_results_test.py new file mode 100755 index 00000000..84c4cb76 --- /dev/null +++ b/pipeline/combine_results_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/python -S +""" +combine_results_test.py: Tests for combine_results.py +""" + +import csv +import cStringIO +import unittest + +import combine_results # module under test + + +# TODO: Make these test more the header row. They rely heavily on the file +# system! + +class CombineResultsTest(unittest.TestCase): + + def testCombineDistResults(self): + stdin = cStringIO.StringIO('') + out = cStringIO.StringIO() + c_out = csv.writer(out) + + combine_results.CombineDistResults(stdin, c_out, 10) + actual = out.getvalue() + self.assert_(actual.startswith('date'), actual) + + def testCombineAssocResults(self): + stdin = cStringIO.StringIO('') + out = cStringIO.StringIO() + c_out = csv.writer(out) + + combine_results.CombineAssocResults(stdin, c_out, 10) + actual = out.getvalue() + self.assert_(actual.startswith('dummy'), actual) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/combine_status.py b/pipeline/combine_status.py new file mode 100755 index 00000000..4fbb36ac --- /dev/null +++ b/pipeline/combine_status.py @@ -0,0 +1,298 @@ +#!/usr/bin/python +"""Summarize the results of many RAPPOR analysis runs. + +Takes a list of STATUS.txt files on stdin, and reads the corresponding spec.txt +and log.txt files. Writes a CSV to stdout. Row key is (metric, date). +""" + +import collections +import csv +import json +import os +import re +import sys + + +# Parse bash 'time' output: +# real 0m11.578s + +# TODO: Parse the time from metrics.json instead. +TIMING_RE = re.compile( + r'real \s+ (\d+) m ([\d.]+) s', re.VERBOSE) + +# TODO: Could have decode-dist and decode-assoc output the PID? +PID_RE = re.compile( + r'write_pid.py: PID (\d+)') # not VERBOSE, spaces are literal + + +def ParseMemCsv(f): + """Compute summary stats for memory. + + vm5_peak_kib -> max(vm_peak_kib) # over 5 second intervals. Since it uses + the kernel, it's accurate except for takes that spike in their last 4 + seconds. + + vm5_mean_kib -> mean(vm_size_kib) # over 5 second intervals + """ + peak_by_pid = collections.defaultdict(list) + size_by_pid = collections.defaultdict(list) + + # Parse columns we care about, by PID + c = csv.reader(f) + for i, row in enumerate(c): + if i == 0: + continue # skip header + # looks like timestamp, pid, then (rss, peak, size) + _, pid, _, peak, size = row + if peak != '': + peak_by_pid[pid].append(int(peak)) + if size != '': + size_by_pid[pid].append(int(size)) + + mem_by_pid = {} + + # Now compute summaries + pids = peak_by_pid.keys() + for pid in pids: + peaks = peak_by_pid[pid] + vm5_peak_kib = max(peaks) + + sizes = size_by_pid[pid] + vm5_mean_kib = sum(sizes) / len(sizes) + + mem_by_pid[pid] = (vm5_peak_kib, vm5_mean_kib) + + return mem_by_pid + + +def CheckJobId(job_id, parts): + """Sanity check for date or smoke test.""" + if not job_id.startswith('201') and not job_id.startswith('smoke'): + raise RuntimeError( + "Expected job ID to start with '201' or 'smoke': got %r (%s)" % + (job_id, parts)) + + +def ReadStatus(f): + status_line = f.readline().strip() + return status_line.split()[0] # OK, TIMEOUT, FAIL + + +def CombineDistTaskStatus(stdin, c_out, mem_by_pid): + """Read status task paths from stdin, write CSV summary to c_out'.""" + + #util.log('%s', mem_by_pid) + + # Parses: + # - input path for metric name and date + # - spec.txt for task params + # - STATUS.txt for task success/failure + # - metrics.json for output metrics + # - log.txt for timing, if it ran to completion + # - and for structured data + # - join with mem by PID + + header = ( + 'job_id', 'params_file', 'map_file', + 'metric', 'date', + 'vm5_peak_kib', 'vm5_mean_kib', # set when not skipped + 'seconds', 'status', + # only set when OK + 'num_reports', 'num_rappor', 'allocated_mass', + # only set when failed + 'fail_reason') + c_out.writerow(header) + + for line in stdin: + # + # Receive a STATUS.txt path on each line of stdin, and parse it. + # + status_path = line.strip() + + with open(status_path) as f: + status = ReadStatus(f) + + # Path should look like this: + # ~/rappor/cron/2015-05-20__19-22-01/raw/Settings.NewTabPage/2015-05-19/STATUS.txt + parts = status_path.split('/') + job_id = parts[-5] + CheckJobId(job_id, parts) + + # + # Parse the job spec + # + result_dir = os.path.dirname(status_path) + spec_file = os.path.join(result_dir, 'spec.txt') + with open(spec_file) as f: + spec_line = f.readline() + # See backfill.sh analyze-one for the order of these 7 fields. + # There are 3 job constants on the front. + (num_reports, metric_name, date, counts_path, params_path, + map_path, _) = spec_line.split() + + # NOTE: These are all constant per metric. Could have another CSV and + # join. But denormalizing is OK for now. + params_file = os.path.basename(params_path) + map_file = os.path.basename(map_path) + + # remove extension + params_file, _ = os.path.splitext(params_file) + map_file, _ = os.path.splitext(map_file) + + # + # Read the log + # + log_file = os.path.join(result_dir, 'log.txt') + with open(log_file) as f: + lines = f.readlines() + + # Search lines in reverse order for total time. It could have output from + # multiple 'time' statements, and we want the last one. + seconds = None # for skipped + for i in xrange(len(lines) - 1, -1, -1): + # TODO: Parse the R timing too. Could use LOG_RECORD_RE. + m = TIMING_RE.search(lines[i]) + if m: + min_part, sec_part = m.groups() + seconds = float(min_part) * 60 + float(sec_part) + break + + # Extract stack trace + if status == 'FAIL': + # Stack trace looks like: "Calls: main -> RunOne ..." + fail_reason = ''.join(line.strip() for line in lines if 'Calls' in line) + else: + fail_reason = None + + # Extract PID and join with memory results + pid = None + vm5_peak_kib = None + vm5_mean_kib = None + if mem_by_pid: + for line in lines: + m = PID_RE.match(line) + if m: + pid = m.group(1) + # Could the PID not exist if the process was super short was less + # than 5 seconds? + try: + vm5_peak_kib, vm5_mean_kib = mem_by_pid[pid] + except KeyError: # sometimes we don't add mem-track on the front + vm5_peak_kib, vm5_mean_kib = None, None + break + else: + pass # we weren't passed memory.csv + + # + # Read the metrics + # + metrics = {} + metrics_file = os.path.join(result_dir, 'metrics.json') + if os.path.isfile(metrics_file): + with open(metrics_file) as f: + metrics = json.load(f) + + num_rappor = metrics.get('num_detected') + allocated_mass = metrics.get('allocated_mass') + + # Construct and write row + row = ( + job_id, params_file, map_file, + metric_name, date, + vm5_peak_kib, vm5_mean_kib, + seconds, status, + num_reports, num_rappor, allocated_mass, + fail_reason) + + c_out.writerow(row) + + +def CombineAssocTaskStatus(stdin, c_out): + """Read status task paths from stdin, write CSV summary to c_out'.""" + + header = ( + 'job_id', 'metric', 'date', 'status', 'num_reports', + 'total_elapsed_seconds', 'em_elapsed_seconds', 'var1', 'var2', 'd1', + 'd2') + + c_out.writerow(header) + + for line in stdin: + status_path = line.strip() + + with open(status_path) as f: + status = ReadStatus(f) + + parts = status_path.split('/') + job_id = parts[-6] + CheckJobId(job_id, parts) + + # + # Parse the job spec + # + result_dir = os.path.dirname(status_path) + spec_file = os.path.join(result_dir, 'assoc-spec.txt') + with open(spec_file) as f: + spec_line = f.readline() + # See backfill.sh analyze-one for the order of these 7 fields. + # There are 3 job constants on the front. + + # 5 job params + (_, _, _, _, _, + dummy_num_reports, metric_name, date, reports, var1, var2, map1, + output_dir) = spec_line.split() + + # + # Parse decode-assoc metrics + # + metrics = {} + metrics_file = os.path.join(result_dir, 'assoc-metrics.json') + if os.path.isfile(metrics_file): + with open(metrics_file) as f: + metrics = json.load(f) + + # After we run it we have the actual number of reports + num_reports = metrics.get('num_reports') + total_elapsed_seconds = metrics.get('total_elapsed_time') + em_elapsed_seconds = metrics.get('em_elapsed_time') + estimate_dimensions = metrics.get('estimate_dimensions') + if estimate_dimensions: + d1, d2 = estimate_dimensions + else: + d1, d2 = (0, 0) # unknown + + row = ( + job_id, metric_name, date, status, num_reports, total_elapsed_seconds, + em_elapsed_seconds, var1, var2, d1, d2) + c_out.writerow(row) + + +def main(argv): + action = argv[1] + + try: + mem_csv = argv[2] + except IndexError: + mem_by_pid = None + else: + with open(mem_csv) as f: + mem_by_pid = ParseMemCsv(f) + + if action == 'dist': + c_out = csv.writer(sys.stdout) + CombineDistTaskStatus(sys.stdin, c_out, mem_by_pid) + + elif action == 'assoc': + c_out = csv.writer(sys.stdout) + CombineAssocTaskStatus(sys.stdin, c_out) + + else: + raise RuntimeError('Invalid action %r' % action) + + +if __name__ == '__main__': + try: + main(sys.argv) + except RuntimeError, e: + print >>sys.stderr, 'FATAL: %s' % e + sys.exit(1) diff --git a/pipeline/combine_status_test.py b/pipeline/combine_status_test.py new file mode 100755 index 00000000..4606587e --- /dev/null +++ b/pipeline/combine_status_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/python -S +""" +combine_status_test.py: Tests for combine_status.py +""" + +import csv +import cStringIO +import unittest + +import combine_status # module under test + + +# TODO: Make these test more the header row. They rely heavily on the file +# system! + +class CombineStatusTest(unittest.TestCase): + + def testCombineDistTaskStatus(self): + stdin = cStringIO.StringIO('') + out = cStringIO.StringIO() + c_out = csv.writer(out) + + combine_status.CombineDistTaskStatus(stdin, c_out, {}) + actual = out.getvalue() + self.assert_(actual.startswith('job_id,params_file,'), actual) + + def testCombineAssocTaskStatus(self): + stdin = cStringIO.StringIO('') + out = cStringIO.StringIO() + c_out = csv.writer(out) + + combine_status.CombineAssocTaskStatus(stdin, c_out) + actual = out.getvalue() + self.assert_(actual.startswith('job_id,metric,'), actual) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/cook.sh b/pipeline/cook.sh new file mode 100755 index 00000000..e820d440 --- /dev/null +++ b/pipeline/cook.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Take the raw data from the analysis and massage it into various formats +# suitable for display. +# +# Usage: +# ./cook.sh + +set -o nounset +set -o pipefail +set -o errexit + +readonly THIS_DIR=$(dirname $0) +readonly RAPPOR_SRC=$(cd $THIS_DIR/.. && pwd) + +source $RAPPOR_SRC/pipeline/tools-lib.sh + + +status-files() { + local dir=$1 + find $dir -name STATUS.txt +} + +results-files() { + local dir=$1 + find $dir -name results.csv +} + +count-results() { + # first field of each line is one of {OK, TIMEOUT, FAIL, SKIPPED} + status-files "$@" \ + | xargs cat \ + | cut -d ' ' -f 1 \ + | sort | uniq -c | sort -n -r +} + +# +# For dist cron job +# + +# Combine status of tasks over multiple jobs. Each row is a task (decode-dist +# invocation). This has the number of reports. +combine-dist-task-status() { + local base_dir=${1:-~/rappor/cron} + local job_dir=${2:-~/rappor/cron/2015-05-22__05-58-01} + + local out=$job_dir/task-status.csv + + # Ignore memory for now. + time status-files $base_dir | TOOLS-combine-status dist > $out + echo "Wrote $out" +} + +# Create a single dist.csv time series for a GIVEN metric. +combine-dist-results-one() { + local base_dir=$1 + local job_dir=$2 + local metric_name=$3 + #echo FOO $base_dir $metric_name + + local out_dir=$job_dir/cooked/$metric_name + mkdir -p $out_dir + + # Glob to capture this specific metric name over ALL job IDs. + find $base_dir/*/raw/$metric_name -name STATUS.txt \ + | TOOLS-combine-results dist 5 \ + > $out_dir/dist.csv +} + +# Creates a dist.csv file for EACH metric. TODO: Rename one/many +combine-dist-results() { + local base_dir=${1:-~/rappor/cron} + local job_dir=${2:-~/rappor/cron/2015-05-22__05-58-01} + + # Direct subdirs of 'raw' are metrics. Just print filename. + find $base_dir/*/raw -mindepth 1 -maxdepth 1 -type d -a -printf '%f\n' \ + | sort | uniq \ + | xargs --verbose -n1 -- \ + $0 combine-dist-results-one $base_dir $job_dir +} + +# Take the task-status.csv file, which has row key (metric, date). Writes +# num_reports.csv and status.csv per metric, and a single overview.csv for all +# metrics. +dist-metric-status() { + local job_dir=${1:-_tmp/results-10} + local out_dir=$job_dir/cooked + + TOOLS-metric-status dist $job_dir/task-status.csv $out_dir +} + +# +# For association analysis cron job +# + +combine-assoc-task-status() { + local base_dir=${1:-~/rappor/chrome-assoc-smoke} + local job_dir=${2:-$base_dir/smoke1} + + local out=$job_dir/assoc-task-status.csv + + time find $base_dir -name assoc-status.txt \ + | TOOLS-combine-status assoc \ + > $out + + echo "Wrote $out" +} + +# Create a single assoc.csv time series for a GIVEN (var1, var2) pair. +combine-assoc-results-one() { + local base_dir=$1 + local job_dir=$2 + local metric_pair_rel_path=$3 + + local out_dir=$job_dir/cooked/$metric_pair_rel_path + mkdir -p $out_dir + + # Glob to capture this specific metric name over ALL job IDs. + find $base_dir/*/raw/$metric_pair_rel_path -name assoc-status.txt \ + | TOOLS-combine-results assoc 5 \ + > $out_dir/assoc-results-series.csv +} + +# Creates a dist.csv file for EACH metric. TODO: Rename one/many +combine-assoc-results() { + local base_dir=${1:-~/rappor/chrome-assoc-smoke} + local job_dir=${2:-$base_dir/smoke3} + + # Direct subdirs of 'raw' are metrics, and subdirs of that are variable + # pairs. Print "$metric_name/$pair_name". + find $base_dir/*/raw -mindepth 2 -maxdepth 2 -type d -a -printf '%P\n' \ + | sort | uniq \ + | xargs --verbose -n1 -- \ + $0 combine-assoc-results-one $base_dir $job_dir +} + +# Take the assoc-task-status.csv file, which has row key (metric, date). Writes +# num_reports.csv and status.csv per metric, and a single overview.csv for all +# metrics. +assoc-metric-status() { + local job_dir=${1:-~/rappor/chrome-assoc-smoke/smoke3} + local out_dir=$job_dir/cooked + + TOOLS-metric-status assoc $job_dir/assoc-task-status.csv $out_dir +} + +"$@" diff --git a/pipeline/csv-to-html-test.sh b/pipeline/csv-to-html-test.sh new file mode 100755 index 00000000..754d083f --- /dev/null +++ b/pipeline/csv-to-html-test.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Test for csv_to_html.py. +# +# Usage: +# ./csv-to-html-test.sh + +set -o nounset +set -o pipefail +set -o errexit + +test-basic() { + ./csv_to_html.py <{b}' <{v}' < placeholders instead of . + p.add_option( + '--table', dest='table', default=False, action='store_true', + help='Add
tags (useful for testing)') + + return p + + +def ParseSpec(arg_list): + """Given an argument list, return a string -> string dictionary.""" + # The format string is passed the cell value. Escaped as HTML? + d = {} + for s in arg_list: + try: + name, value = s.split(' ', 1) + except ValueError: + raise RuntimeError('Invalid column format %r' % s) + d[name] = value + return d + + +def PrintRow(row, col_names, col_formats, defs, percent_cols): + """Print a CSV row as HTML, using the given formatting. + + Returns: + An array of booleans indicating whether each cell is a number. + """ + is_number_flags = [False] * len(col_names) + + for i, cell in enumerate(row): + # The cell as a string. By default we leave it as is; it may be mutated + # below. + cell_str = cell + css_class = '' # CSS class for the cell. + col_name = col_names[i] # column that the cell is under + + # Does the cell look like a float? + try: + cell_float = float(cell) + if col_name in percent_cols: # Floats can be formatted as percentages. + cell_str = '{:.1f}%'.format(cell_float * 100) + else: + # Arbitrarily use 3 digits of precision for display + cell_str = '{:.3f}'.format(cell_float) + css_class = 'num' + is_number_flags[i] = True + except ValueError: + pass + + # Does it look lik an int? + try: + cell_int = int(cell) + cell_str = '{:,}'.format(cell_int) + css_class = 'num' + is_number_flags[i] = True + except ValueError: + pass + + # Special CSS class for R NA values. + if cell_str.strip() == 'NA': + css_class = 'num na' # num should right justify; na should make it red + is_number_flags[i] = True + + if css_class: + print ' '.format(css_class), + else: + print ' ', + + cell_safe = cgi.escape(cell_str) + + # If the cell has a format string, print it this way. + + fmt = col_formats.get(col_name) # e.g. "../{date}.html" + if fmt: + # Copy variable bindings + bindings = dict(defs) + + # Also let the format string use other column names. TODO: Is there a + # more efficient way? + bindings.update(zip(col_names, [cgi.escape(c) for c in row])) + + bindings[col_name] = cell_safe + + print fmt.format(**bindings), # no newline + else: + print cell_safe, # no newline + + print '' + + return is_number_flags + + +def ReadCsv(f): + """Read the CSV file, returning the column names and rows.""" + c = csv.reader(f) + + # The first row of the CSV is assumed to be a header. The rest are data. + col_names = [] + rows = [] + for i, row in enumerate(c): + if i == 0: + col_names = row + continue + rows.append(row) + return col_names, rows + + +def PrintColGroup(col_names, col_is_numeric): + """Print HTML colgroup element, used for JavaScript sorting.""" + print '' + for i, col in enumerate(col_names): + # CSS class is used for sorting + if col_is_numeric[i]: + css_class = 'number' + else: + css_class = 'case-insensitive' + + # NOTE: id is a comment only; not used + print ' '.format(col, css_class) + print '' + + +def main(argv): + (opts, argv) = CreateOptionsParser().parse_args(argv) + + col_formats = ParseSpec(opts.col_formats) + defs = ParseSpec(opts.defs) + + col_names, rows = ReadCsv(sys.stdin) + + for col in opts.percent_cols: + if col not in col_names: + raise RuntimeError('--percent-col %s is not a valid column' % col) + + # By default, we don't print the bit -- that's up to the host page + if opts.table: + print '
' + + print '' + for col in col_names: + # change _ to space so long column names can wrap + print ' ' % cgi.escape(col.replace('_', ' ')) + print '' + + # Assume all columns are numeric at first. Look at each row for non-numeric + # values. + col_is_numeric = [True] * len(col_names) + + print '' + for row in rows: + print ' ' + is_number_flags = PrintRow(row, col_names, col_formats, defs, + opts.percent_cols) + + # If one cell in a column is not a number, then the whole cell isn't. + for (i, is_number) in enumerate(is_number_flags): + if not is_number: + col_is_numeric[i] = False + + print ' ' + print '' + + PrintColGroup(col_names, col_is_numeric) + + if opts.table: + print '
%s
' + + +if __name__ == '__main__': + try: + main(sys.argv) + except RuntimeError, e: + print >>sys.stderr, 'FATAL: %s' % e + sys.exit(1) diff --git a/pipeline/csv_to_html_test.py b/pipeline/csv_to_html_test.py new file mode 100755 index 00000000..5fd5822a --- /dev/null +++ b/pipeline/csv_to_html_test.py @@ -0,0 +1,24 @@ +#!/usr/bin/python -S +""" +csv_to_html_test.py: Tests for csv_to_html.py +""" + +import unittest + +import csv_to_html # module under test + + +class CsvToHtmlTest(unittest.TestCase): + + def testParseSpec(self): + self.assertEqual( + {'foo': 'bar', 'spam': 'eggs'}, + csv_to_html.ParseSpec(['foo bar', 'spam eggs'])) + + self.assertEqual( + {}, + csv_to_html.ParseSpec([])) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/dist.sh b/pipeline/dist.sh new file mode 100755 index 00000000..14c76e34 --- /dev/null +++ b/pipeline/dist.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Usage: +# ./dist.sh + +set -o nounset +set -o pipefail +set -o errexit + +readonly THIS_DIR=$(dirname $0) +readonly RAPPOR_SRC=$(cd $THIS_DIR/.. && pwd) + +source $RAPPOR_SRC/util.sh # log, banner +source $RAPPOR_SRC/pipeline/tools-lib.sh +source $RAPPOR_SRC/pipeline/alarm-lib.sh + +readonly NUM_ARGS=7 # used for xargs + +decode-dist-one() { + # Job constants + local rappor_src=$1 + local timeout_secs=$2 + local min_reports=$3 + shift 3 # job constants do not vary per task and are not part of the spec + + # 7 spec variables + local num_reports=$1 # unused, only for filtering + local metric_name=$2 + local date=$3 + local counts=$4 + local params=$5 + local map=$6 + local results_dir=$7 + + local task_dir=$results_dir/$metric_name/$date + mkdir --verbose -p $task_dir + + local log_file=$task_dir/log.txt + local status_file=$task_dir/STATUS.txt + + # Record the spec so we know params, counts, etc. + echo "$@" > $task_dir/spec.txt + + if test $num_reports -lt $min_reports; then + local msg="SKIPPED because $num_reports reports is less than $min_reports" + # Duplicate this message + echo "$msg" > $status_file + echo "$msg" > $log_file + return + fi + + # Run it with a timeout, and record status in the task dir. + { time \ + alarm-status $status_file $timeout_secs \ + $rappor_src/bin/decode-dist \ + --counts $counts \ + --params $params \ + --map $map \ + --output-dir $task_dir + } >$log_file 2>&1 + + # TODO: set output name instead of dir? + # Right now it's the only CSV file. +} + +# Print the number of processes to use. +# NOTE: This is copied from google/rappor regtest.sh. +# It also doesn't take into account the fact that we are memory-bound. +# +# 128 GiB / 4GiB would also imply about 32 processes though. +num-processes() { + local processors=$(grep -c ^processor /proc/cpuinfo || echo 4) + if test $processors -gt 1; then # leave one CPU for the OS + processors=$(expr $processors - 1) + fi + echo $processors +} + +#readonly DEFAULT_MAX_PROCS=6 # for andychu2.hot, to avoid locking up UI +#readonly DEFAULT_MAX_PROCS=16 # for rappor-ac.hot, to avoid thrashing +readonly DEFAULT_MAX_PROCS=$(num-processes) + +#readonly DEFAULT_MAX_TASKS=12 +readonly DEFAULT_MAX_TASKS=10000 # more than the max + +# NOTE: Since we have 125 GB RAM, and processes can take up to 12 gigs of RAM, +# only use parallelism of 10, even though we have 31 cores. + +readonly DEFAULT_MIN_REPORTS=5000 + + +decode-dist-many() { + local job_dir=$1 + local spec_list=$2 + local timeout_secs=${3:-1200} # default timeout + local max_procs=${4:-$DEFAULT_MAX_PROCS} + local rappor_src=${5:-$RAPPOR_SRC} + local min_reports=${6:-$DEFAULT_MIN_REPORTS} + + local interval_secs=5 + local pid_dir="$job_dir/pids" + local sys_mem="$job_dir/system-mem.csv" + mkdir --verbose -p $pid_dir + + time cat $spec_list \ + | xargs --verbose -n $NUM_ARGS -P $max_procs --no-run-if-empty -- \ + $0 decode-dist-one $rappor_src $timeout_secs $min_reports +} + +# Combine/summarize results and task metadata from the parallel decode-dist +# processes. Render them as HTML. +combine-and-render-html() { + local jobs_base_dir=$1 + local job_dir=$2 + + banner "Combining dist task status" + TOOLS-cook combine-dist-task-status $jobs_base_dir $job_dir + + banner "Combining dist results" + TOOLS-cook combine-dist-results $jobs_base_dir $job_dir + + banner "Splitting out status per metric, and writing overview" + TOOLS-cook dist-metric-status $job_dir + + # The task-status.csv file should have the a JOB ID. + banner "Building overview.html and per-metric HTML" + TOOLS-gen-ui build-html1 $job_dir + + banner "Building individual results.html (for ONE day)" + TOOLS-gen-ui results-html $job_dir +} + +"$@" diff --git a/pipeline/metric_status.R b/pipeline/metric_status.R new file mode 100755 index 00000000..0774423a --- /dev/null +++ b/pipeline/metric_status.R @@ -0,0 +1,343 @@ +#!/usr/bin/Rscript +# +# Write an overview of task status, per-metric task status, task histograms. + +library(data.table) +library(ggplot2) + +options(stringsAsFactors = FALSE) # get rid of annoying behavior + +Log <- function(fmt, ...) { + cat(sprintf(fmt, ...)) + cat('\n') +} + +# max of non-NA values; NA if there are none +MaybeMax <- function(values) { + v <- values[!is.na(values)] + if (length(v) == 0) { + m <- NA + } else { + m <- max(v) + } + as.numeric(m) # data.table requires this; otherwise we get type errors +} + +# mean of non-NA values; NA if there are none +MaybeMean <- function(values) { + v <- values[!is.na(values)] + if (length(v) == 0) { + m <- NA + } else { + m <- mean(v) + } + as.numeric(m) # data.table require this; otherwise we get type errors +} + +WriteDistOverview <- function(summary, output_dir) { + s <- data.table(summary) # data.table syntax is easier here + + by_metric <- s[ , list( + params_file = unique(params_file), + map_file = unique(map_file), + days = length(date), + max_num_reports = MaybeMax(num_reports), + + # summarize status + ok = sum(status == 'OK'), + fail = sum(status == 'FAIL'), + timeout = sum(status == 'TIMEOUT'), + skipped = sum(status == 'SKIPPED'), + + # TODO: Need to document the meaning of these metrics. + # All could be NA + # KiB -> MB + #max_vm5_peak_mb = MaybeMax(vm5_peak_kib * 1024 / 1e6), + #mean_vm5_mean_mb = MaybeMean(vm5_mean_kib * 1024 / 1e6), + + mean_secs = MaybeMean(seconds), + mean_allocated_mass = MaybeMean(allocated_mass) + + # unique failure reasons + # This can be used when there are different call stacks. + #fail_reasons = length(unique(fail_reason[fail_reason != ""])) + ), by=metric] + + # Case insensitive sort by metric name + by_metric <- by_metric[order(tolower(by_metric$metric)), ] + + overview_path <- file.path(output_dir, 'overview.csv') + write.csv(by_metric, file = overview_path, row.names = FALSE) + Log("Wrote %s", overview_path) + + by_metric +} + +WriteDistMetricStatus <- function(summary, output_dir) { + # Write status.csv, num_reports.csv, and mass.csv for each metric. + + s <- data.table(summary) + + # loop over unique metrics, and write a CSV for each one + for (m in unique(s$metric)) { + # Select cols, and convert units. Don't need params / map / metric. + subframe <- s[s$metric == m, + list(job_id, date, status, + #vm5_peak_mb = vm5_peak_kib * 1024 / 1e6, + #vm5_mean_mb = vm5_mean_kib * 1024 / 1e6, + num_reports, + seconds, + allocated_mass, num_rappor)] + + # Sort by descending date. Alphabetical sort works fine for YYYY-MM-DD. + subframe <- subframe[order(subframe$date, decreasing = TRUE), ] + + out_path = file.path(output_dir, m, 'status.csv') + write.csv(subframe, file = out_path, row.names = FALSE) + Log("Wrote %s", out_path) + } + + # This one is just for plotting with dygraphs. TODO: can dygraphs do + # something smarter? Maybe you need to select the column in JavaScript, and + # pass it an array, rather than CSV text. + for (m in unique(s$metric)) { + f1 <- s[s$metric == m, list(date, num_reports)] + path1 <- file.path(output_dir, m, 'num_reports.csv') + # NOTE: dygraphs (only in Firefox?) doesn't like the quotes around + # "2015-04-03". In general, we can't turn off quotes, because strings with + # double quotes will be invalid CSV files. But in this case, we only have + # date and number columns, so we can. dygraphs is mistaken here. + write.csv(f1, file = path1, row.names = FALSE, quote = FALSE) + Log("Wrote %s", path1) + + # Write unallocated mass. TODO: Write the other 2 vars too? + f2 <- s[s$metric == m, + list(date, + unallocated_mass = 1.0 - allocated_mass)] + + path2 <- file.path(output_dir, m, 'mass.csv') + write.csv(f2, file = path2, row.names = FALSE, quote = FALSE) + Log("Wrote %s", path2) + } +} + +WritePlot <- function(p, outdir, filename, width = 800, height = 600) { + filename <- file.path(outdir, filename) + png(filename, width = width, height = height) + plot(p) + dev.off() + Log('Wrote %s', filename) +} + +# Make sure the histogram has some valid input. If we don't do this, ggplot +# blows up with an unintuitive error message. +CheckHistogramInput <- function(v) { + if (all(is.na(v))) { + arg_name <- deparse(substitute(v)) # R idiom to get name + Log('FATAL: All values in %s are NA (no successful runs?)', arg_name) + quit(status = 1) + } +} + +WriteDistHistograms <- function(s, output_dir) { + CheckHistogramInput(s$allocated_mass) + + p <- qplot(s$allocated_mass, geom = "histogram") + t <- ggtitle("Allocated Mass by Task") + x <- xlab("allocated mass") + y <- ylab("number of tasks") + WritePlot(p + t + x + y, output_dir, 'allocated_mass.png') + + CheckHistogramInput(s$num_rappor) + + p <- qplot(s$num_rappor, geom = "histogram") + t <- ggtitle("Detected Strings by Task") + x <- xlab("detected strings") + y <- ylab("number of tasks") + WritePlot(p + t + x + y, output_dir, 'num_rappor.png') + + CheckHistogramInput(s$num_reports) + + p <- qplot(s$num_reports / 1e6, geom = "histogram") + t <- ggtitle("Raw Reports by Task") + x <- xlab("millions of reports") + y <- ylab("number of tasks") + WritePlot(p + t + x + y, output_dir, 'num_reports.png') + + CheckHistogramInput(s$seconds) + + p <- qplot(s$seconds, geom = "histogram") + t <- ggtitle("Analysis Duration by Task") + x <- xlab("seconds") + y <- ylab("number of tasks") + WritePlot(p + t + x + y, output_dir, 'seconds.png') + + # NOTE: Skipping this for 'series' jobs. + if (sum(!is.na(s$vm5_peak_kib)) > 0) { + p <- qplot(s$vm5_peak_kib * 1024 / 1e6, geom = "histogram") + t <- ggtitle("Peak Memory Usage by Task") + x <- xlab("Peak megabytes (1e6 bytes) of memory") + y <- ylab("number of tasks") + WritePlot(p + t + x + y, output_dir, 'memory.png') + } +} + +ProcessAllDist <- function(s, output_dir) { + Log('dist: Writing per-metric status.csv') + WriteDistMetricStatus(s, output_dir) + + Log('dist: Writing histograms') + WriteDistHistograms(s, output_dir) + + Log('dist: Writing aggregated overview.csv') + WriteDistOverview(s, output_dir) +} + +# Write the single CSV file loaded by assoc-overview.html. +WriteAssocOverview <- function(summary, output_dir) { + s <- data.table(summary) # data.table syntax is easier here + + by_metric <- s[ , list( + #params_file = unique(params_file), + #map_file = unique(map_file), + + days = length(date), + max_num_reports = MaybeMax(num_reports), + + # summarize status + ok = sum(status == 'OK'), + fail = sum(status == 'FAIL'), + timeout = sum(status == 'TIMEOUT'), + skipped = sum(status == 'SKIPPED'), + + mean_total_secs = MaybeMean(total_elapsed_seconds), + mean_em_secs = MaybeMean(em_elapsed_seconds) + + ), by=list(metric)] + + # Case insensitive sort by metric name + by_metric <- by_metric[order(tolower(by_metric$metric)), ] + + overview_path <- file.path(output_dir, 'assoc-overview.csv') + write.csv(by_metric, file = overview_path, row.names = FALSE) + Log("Wrote %s", overview_path) + + by_metric +} + +# Write the CSV files loaded by assoc-metric.html -- that is, one +# metric-status.csv for each metric name. +WriteAssocMetricStatus <- function(summary, output_dir) { + s <- data.table(summary) + csv_list <- unique(s[, list(metric)]) + for (i in 1:nrow(csv_list)) { + u <- csv_list[i, ] + # Select cols, and convert units. Don't need params / map / metric. + by_pair <- s[s$metric == u$metric, + list(days = length(date), + max_num_reports = MaybeMax(num_reports), + + # summarize status + ok = sum(status == 'OK'), + fail = sum(status == 'FAIL'), + timeout = sum(status == 'TIMEOUT'), + skipped = sum(status == 'SKIPPED'), + + mean_total_secs = MaybeMean(total_elapsed_seconds), + mean_em_secs = MaybeMean(em_elapsed_seconds) + ), + by=list(var1, var2)] + + # Case insensitive sort by var1 name + by_pair <- by_pair[order(tolower(by_pair$var1)), ] + + csv_path <- file.path(output_dir, u$metric, 'metric-status.csv') + write.csv(by_pair, file = csv_path, row.names = FALSE) + Log("Wrote %s", csv_path) + } +} + +# This naming convention is in task_spec.py AssocTaskSpec. +FormatAssocRelPath <- function(metric, var1, var2) { + v2 <- gsub('..', '_', var2, fixed = TRUE) + var_dir <- sprintf('%s_X_%s', var1, v2) + file.path(metric, var_dir) +} + +# Write the CSV files loaded by assoc-pair.html -- that is, one pair-status.csv +# for each (metric, var1, var2) pair. +WriteAssocPairStatus <- function(summary, output_dir) { + + s <- data.table(summary) + + csv_list <- unique(s[, list(metric, var1, var2)]) + Log('CSV list:') + print(csv_list) + + # loop over unique metrics, and write a CSV for each one + for (i in 1:nrow(csv_list)) { + u <- csv_list[i, ] + + # Select cols, and convert units. Don't need params / map / metric. + subframe <- s[s$metric == u$metric & s$var1 == u$var1 & s$var2 == u$var2, + list(job_id, date, status, + num_reports, d1, d2, + total_elapsed_seconds, + em_elapsed_seconds)] + + # Sort by descending date. Alphabetical sort works fine for YYYY-MM-DD. + subframe <- subframe[order(subframe$date, decreasing = TRUE), ] + + pair_rel_path <- FormatAssocRelPath(u$metric, u$var1, u$var2) + + csv_path <- file.path(output_dir, pair_rel_path, 'pair-status.csv') + write.csv(subframe, file = csv_path, row.names = FALSE) + Log("Wrote %s", csv_path) + + # Write a file with the raw variable names. Parsed by ui.sh, to pass to + # csv_to_html.py. + meta_path <- file.path(output_dir, pair_rel_path, 'pair-metadata.txt') + + # NOTE: The conversion from data.table to character vector requires + # stringsAsFactors to work correctly! + lines <- as.character(u) + writeLines(lines, con = meta_path) + Log("Wrote %s", meta_path) + } +} + +ProcessAllAssoc <- function(s, output_dir) { + Log('assoc: Writing pair-status.csv for each variable pair in each metric') + WriteAssocPairStatus(s, output_dir) + + Log('assoc: Writing metric-status.csv for each metric') + WriteAssocMetricStatus(s, output_dir) + + Log('assoc: Writing aggregated overview.csv') + WriteAssocOverview(s, output_dir) +} + +main <- function(argv) { + # increase ggplot font size globally + theme_set(theme_grey(base_size = 16)) + + action = argv[[1]] + input = argv[[2]] + output_dir = argv[[3]] + + if (action == 'dist') { + summary = read.csv(input) + ProcessAllDist(summary, output_dir) + } else if (action == 'assoc') { + summary = read.csv(input) + ProcessAllAssoc(summary, output_dir) + } else { + stop(sprintf('Invalid action %s', action)) + } + + Log('Done') +} + +if (length(sys.frames()) == 0) { + main(commandArgs(TRUE)) +} diff --git a/pipeline/regtest.sh b/pipeline/regtest.sh new file mode 100755 index 00000000..a29a0f08 --- /dev/null +++ b/pipeline/regtest.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# End-to-end tests for the dashboard. +# +# Usage: +# ./regtest.sh +# +# NOTE: Must be run in this directory (rappor/pipeline). + +set -o nounset +set -o pipefail +set -o errexit + +# Create schema and params. +create-metadata() { + mkdir -p _tmp/metadata + echo 'Hello from regtest.sh' + + local params_path=_tmp/metadata/regtest_params.csv + + # Relying on $RAPPOR_SRC/regtest.sh + cp --verbose ../_tmp/python/demo1/case_params.csv $params_path + + # For now, use the same map everywhere. + cat >_tmp/metadata/dist-analysis.csv <_tmp/metadata/rappor-vars.csv < map to analyze against. + + TODO: Support a LIST of maps. Users should be able to specify more than one. + """ + def __init__(self, f, map_dir): + self.dist_maps = _ReadDistMaps(f) + self.map_dir = map_dir + + def GetMapPath(self, var_name): + filename = self.dist_maps[var_name] + return os.path.join(self.map_dir, filename) + + +def CreateFieldIdLookup(f): + """Create a dictionary that specifies single variable analysis each var. + + Args: + config_dir: directory of metadata, output by update_rappor.par + + Returns: + A dictionary from field ID -> full field name + + NOTE: Right now we're only doing single variable analysis for strings, so we + don't have the "type". + """ + field_id_lookup = {} + c = csv.reader(f) + for i, row in enumerate(c): + if i == 0: + expected = ['metric', 'field', 'field_type', 'params', 'field_id'] + if row != expected: + raise RuntimeError('Expected CSV header %s' % expected) + continue + + metric, field, field_type, _, field_id = row + + if field_type != 'string': + continue + + # Paper over the difference between plain metrics (single variable) and + # metrics with fields (multiple variables, for association analysis). + if field: + full_field_name = '%s.%s' % (metric, field) + else: + full_field_name = metric + + field_id_lookup[field_id] = full_field_name + return field_id_lookup + + +def _ReadVarSchema(f): + """Given the rappor-vars.csv file, return a list of metric/var/type.""" + # metric -> list of (variable name, type) + assoc_metrics = collections.defaultdict(list) + params_lookup = {} + + c = csv.reader(f) + for i, row in enumerate(c): + if i == 0: + expected = ['metric', 'var', 'var_type', 'params'] + if row != expected: + raise RuntimeError('Expected CSV header %s, got %s' % (expected, row)) + continue + + metric, var, var_type, params = row + if var == '': + full_var_name = metric + else: + full_var_name = '%s.%s' % (metric, var) + # Also group multi-dimensional reports + assoc_metrics[metric].append((var, var_type)) + + params_lookup[full_var_name] = params + + return assoc_metrics, params_lookup + + +class VarSchema(object): + """Object representing rappor-vars.csv. + + Right now we use it for slightly different purposes for dist and assoc + analysis. + """ + def __init__(self, f, params_dir): + self.assoc_metrics, self.params_lookup = _ReadVarSchema(f) + self.params_dir = params_dir + + def GetParamsPath(self, var_name): + filename = self.params_lookup[var_name] + return os.path.join(self.params_dir, filename + '.csv') + + def GetAssocMetrics(self): + return self.assoc_metrics + + +def CountReports(f): + num_reports = 0 + for line in f: + first_col = line.split(',')[0] + num_reports += int(first_col) + return num_reports + + +DIST_INPUT_PATH_RE = re.compile(r'.*/(\d+-\d+-\d+)/(\S+)_counts.csv') + + +def DistInputIter(stdin): + """Read lines from stdin and extract fields to construct analysis tasks.""" + for line in stdin: + m = DIST_INPUT_PATH_RE.match(line) + if not m: + raise RuntimeError('Invalid path %r' % line) + + counts_path = line.strip() + date, field_id = m.groups() + + yield counts_path, date, field_id + + +def DistTaskSpec(input_iter, field_id_lookup, var_schema, dist_maps, bad_c): + """Print task spec for single variable RAPPOR to stdout.""" + + num_bad = 0 + unique_ids = set() + + for counts_path, date, field_id in input_iter: + unique_ids.add(field_id) + + # num_reports is used for filtering + with open(counts_path) as f: + num_reports = CountReports(f) + + # Look up field name from field ID + if field_id_lookup: + field_name = field_id_lookup.get(field_id) + if field_name is None: + # The metric id is the md5 hash of the name. We can miss some, e.g. due + # to debug builds. + if bad_c: + bad_c.writerow((date, field_id, num_reports)) + num_bad += 1 + continue + else: + field_name = field_id + + # NOTE: We could remove the params from the spec if decode_dist.R took the + # --schema flag. The var type is there too. + params_path = var_schema.GetParamsPath(field_name) + map_path= dist_maps.GetMapPath(field_name) + + yield num_reports, field_name, date, counts_path, params_path, map_path + + util.log('%d unique field IDs', len(unique_ids)) + if num_bad: + util.log('Failed field ID -> field name lookup on %d files ' + '(check --field-ids file)', num_bad) + + +ASSOC_INPUT_PATH_RE = re.compile(r'.*/(\d+-\d+-\d+)/(\S+)_reports.csv') + + +def AssocInputIter(stdin): + """Read lines from stdin and extract fields to construct analysis tasks.""" + for line in stdin: + m = ASSOC_INPUT_PATH_RE.match(line) + if not m: + raise RuntimeError('Invalid path %r' % line) + + reports_path = line.strip() + date, metric_name = m.groups() + + yield reports_path, date, metric_name + + +def CreateAssocVarPairs(rappor_metrics): + """Yield a list of pairs of variables that should be associated. + + For now just do all (string x boolean) analysis. + """ + var_pairs = collections.defaultdict(list) + + for metric, var_list in rappor_metrics.iteritems(): + string_vars = [] + boolean_vars = [] + + # Separate variables into strings and booleans + for var_name, var_type in var_list: + if var_type == 'string': + string_vars.append(var_name) + elif var_type == 'boolean': + boolean_vars.append(var_name) + else: + util.log('Unknown type variable type %r', var_type) + + for s in string_vars: + for b in boolean_vars: + var_pairs[metric].append((s, b)) + return var_pairs + + +# For debugging +def PrintAssocVarPairs(var_pairs): + for metric, var_list in var_pairs.iteritems(): + print metric + for var_name, var_type in var_list: + print '\t', var_name, var_type + + +def AssocTaskSpec(input_iter, var_pairs, dist_maps, output_base_dir, bad_c): + """Print the task spec for multiple variable RAPPOR to stdout.""" + # Flow: + # + # Long term: We should have assoc-analysis.xml, next to dist-analysis.xml? + # + # Short term: update_rappor.py should print every combination of string vs. + # bool? Or I guess we have it in rappor-vars.csv + + for reports_path, date, metric_name in input_iter: + pairs = var_pairs[metric_name] + for var1, var2 in pairs: + # Assuming var1 is a string. TODO: Use an assoc file, not dist_maps? + field1_name = '%s.%s' % (metric_name, var1) + map1_path = dist_maps.GetMapPath(field1_name) + + # e.g. domain_X_flags__DID_PROCEED + # Don't use .. in filenames since it could be confusing. + pair_name = '%s_X_%s' % (var1, var2.replace('..', '_')) + output_dir = os.path.join(output_base_dir, metric_name, pair_name, date) + + yield metric_name, date, reports_path, var1, var2, map1_path, output_dir + + +def CreateOptionsParser(): + p = optparse.OptionParser() + + p.add_option( + '--bad-report-out', dest='bad_report', metavar='PATH', type='str', + default='', + help='Optionally write a report of input filenames with invalid field ' + 'IDs to this file.') + p.add_option( + '--config-dir', dest='config_dir', metavar='PATH', type='str', + default='', + help='Directory with metadata schema and params files to read.') + p.add_option( + '--map-dir', dest='map_dir', metavar='PATH', type='str', + default='', + help='Directory with map files to read.') + p.add_option( + '--output-base-dir', dest='output_base_dir', metavar='PATH', type='str', + default='', + help='Root of the directory tree where analysis output will be placed.') + p.add_option( + '--field-ids', dest='field_ids', metavar='PATH', type='str', + default='', + help='Optional CSV file with field IDs (generally should not be used).') + + return p + + +def main(argv): + (opts, argv) = CreateOptionsParser().parse_args(argv) + + if opts.bad_report: + bad_f = open(opts.bad_report, 'w') + bad_c = csv.writer(bad_f) + else: + bad_c = None + + action = argv[1] + + if not opts.config_dir: + raise RuntimeError('--config-dir is required') + if not opts.map_dir: + raise RuntimeError('--map-dir is required') + if not opts.output_base_dir: + raise RuntimeError('--output-base-dir is required') + + # This is shared between the two specs. + path = os.path.join(opts.config_dir, 'dist-analysis.csv') + with open(path) as f: + dist_maps = DistMapLookup(f, opts.map_dir) + + path = os.path.join(opts.config_dir, 'rappor-vars.csv') + with open(path) as f: + var_schema = VarSchema(f, opts.config_dir) + + if action == 'dist': + if opts.field_ids: + with open(opts.field_ids) as f: + field_id_lookup = CreateFieldIdLookup(f) + else: + field_id_lookup = {} + + input_iter = DistInputIter(sys.stdin) + for row in DistTaskSpec(input_iter, field_id_lookup, var_schema, dist_maps, + bad_c): + # The spec is a series of space-separated tokens. + tokens = row + (opts.output_base_dir,) + print ' '.join(str(t) for t in tokens) + + elif action == 'assoc': + # Parse input + input_iter = AssocInputIter(sys.stdin) + + # Create M x N association tasks + var_pairs = CreateAssocVarPairs(var_schema.GetAssocMetrics()) + + # Now add the other constant stuff + for row in AssocTaskSpec( + input_iter, var_pairs, dist_maps, opts.output_base_dir, bad_c): + + num_reports = 0 # placeholder, not filtering yet + tokens = (num_reports,) + row + print ' '.join(str(t) for t in tokens) + + else: + raise RuntimeError('Invalid action %r' % action) + + +if __name__ == '__main__': + try: + main(sys.argv) + except IOError, e: + if e.errno != errno.EPIPE: # ignore broken pipe + raise + except RuntimeError, e: + print >>sys.stderr, 'FATAL: %s' % e + sys.exit(1) diff --git a/pipeline/task_spec_test.py b/pipeline/task_spec_test.py new file mode 100755 index 00000000..94cbac8d --- /dev/null +++ b/pipeline/task_spec_test.py @@ -0,0 +1,61 @@ +#!/usr/bin/python -S +""" +task_spec_test.py: Tests for task_spec.py +""" + +import cStringIO +import unittest + +import task_spec # module under test + + +class TaskSpecTest(unittest.TestCase): + + def testCountReports(self): + f = cStringIO.StringIO("""\ +1,2 +3,4 +5,6 +""") + c = task_spec.CountReports(f) + self.assertEqual(9, c) + + def testDist(self): + # NOTE: These files are opened, in order to count the reports. Maybe skip + # that step. + f = cStringIO.StringIO("""\ +_tmp/counts/2015-12-01/exp_counts.csv +_tmp/counts/2015-12-01/gauss_counts.csv +_tmp/counts/2015-12-02/exp_counts.csv +_tmp/counts/2015-12-02/gauss_counts.csv +""") + input_iter = task_spec.DistInputIter(f) + #for row in input_iter: + # print row + + field_id_lookup = {} + + # var name -> map filename + f = cStringIO.StringIO("""\ +var,map_filename +exp,map.csv +unif,map.csv +gauss,map.csv +""") + dist_maps = task_spec.DistMapLookup(f, '_tmp/maps') + + f2 = cStringIO.StringIO("""\ +metric,var,var_type,params +exp,,string,params +unif,,string,params +gauss,,string,params +""") + var_schema = task_spec.VarSchema(f2, '_tmp/config') + + for row in task_spec.DistTaskSpec( + input_iter, field_id_lookup, var_schema, dist_maps, None): + print row + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/tools-lib.sh b/pipeline/tools-lib.sh new file mode 100644 index 00000000..bb24bc37 --- /dev/null +++ b/pipeline/tools-lib.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Library used to refer to open source tools. + +set -o nounset +set -o pipefail +set -o errexit + +# +# NOTE: RAPPOR_SRC defined by the module that sources (cook.sh or ui.sh) +# + +# These 3 used by cook.sh. + +TOOLS-combine-status() { + $RAPPOR_SRC/pipeline/combine_status.py "$@" +} + +TOOLS-combine-results() { + $RAPPOR_SRC/pipeline/combine_results.py "$@" +} + +TOOLS-metric-status() { + $RAPPOR_SRC/pipeline/metric_status.R "$@" +} + +# Used by ui.sh. + +TOOLS-csv-to-html() { + $RAPPOR_SRC/pipeline/csv_to_html.py "$@" +} + +# +# Higher level scripts +# + +TOOLS-cook() { + $RAPPOR_SRC/pipeline/cook.sh "$@" +} + +# TODO: Rename gen-ui.sh. +TOOLS-gen-ui() { + $RAPPOR_SRC/pipeline/ui.sh "$@" +} + diff --git a/pipeline/ui.sh b/pipeline/ui.sh new file mode 100755 index 00000000..bf351a78 --- /dev/null +++ b/pipeline/ui.sh @@ -0,0 +1,319 @@ +#!/bin/bash +# +# Build the user interface. +# +# Usage: +# ./ui.sh + +set -o nounset +set -o pipefail +set -o errexit + +readonly THIS_DIR=$(dirname $0) +readonly RAPPOR_SRC=$(cd $THIS_DIR/.. && pwd) + +source $RAPPOR_SRC/pipeline/tools-lib.sh + +_link() { + ln --verbose -s -f "$@" +} + +_copy() { + cp --verbose -f "$@" +} + +download-dygraphs() { + local out=third_party + wget --directory $out \ + http://dygraphs.com/1.1.1/dygraph-combined.js +} + +import-table() { + local src=~/git/scratch/ajax/ + cp --verbose $src/table-sort.{js,css} $src/url-hash.js ui + pushd ui + # TODO: Could minify it here + cat table-sort.js url-hash.js > table-lib.js + popd +} + +# Use symlinks so we can edit and reload during development. +symlink-static() { + local kind=$1 + local job_dir=$2 + + local base=$RAPPOR_SRC/ui + + # HTML goes at the top level. + if test "$kind" = dist; then + _link \ + $base/overview.html $base/histograms.html $base/metric.html $base/day.html \ + $job_dir + elif test "$kind" = assoc; then + _link \ + $base/assoc-overview.html $base/assoc-metric.html $base/assoc-pair.html \ + $base/assoc-day.html \ + $job_dir + else + log "Invalid kind $kind" + exit 1 + fi + + mkdir --verbose -p $job_dir/static + + # Static subdir. + _link \ + $base/ui.css $base/ui.js \ + $base/table-sort.css $base/table-lib.js \ + $RAPPOR_SRC/third_party/dygraph-combined.js \ + $job_dir/static +} + + +# Write HTML fragment based on overview.csv. +overview-part-html() { + local job_dir=${1:-_tmp/results-10} + local out=$job_dir/cooked/overview.part.html + # Sort by descending date! + TOOLS-csv-to-html \ + --col-format 'metric {metric}' \ + < $job_dir/cooked/overview.csv \ + > $out + echo "Wrote $out" +} + +metric-part-html() { + local job_dir=${1:-_tmp/results-10} + # Testing it out. This should probably be a different dir. + + for entry in $job_dir/cooked/*; do + # Only do it for dirs + if ! test -d $entry; then + continue + fi + # Now it's a metric dir + echo $entry + + local metric_name=$(basename $entry) + + # Convert status.csv to status.part.html (a fragment) + + # NOTE: counts path could be useful. You need the input tree though. Hash + # it? Or point to the git link. + + # Link to raw CSV + #--col-format 'date {date}' \ + + # TODO: Link to ui/results_viewer.html#{metric}_{date} + # And that needs some JavaScript to load the correct fragment. + # I guess you could do the same with metric.html. Currently it uses a + # symlink. + + # Before job ID: + # --col-format 'date {date}' \ + # --col-format 'status {status}' \ + + local fmt1='date {date}' + local fmt2='status {status}' + + TOOLS-csv-to-html \ + --def "metric $metric_name" \ + --col-format "$fmt1" \ + --col-format "$fmt2" \ + < $entry/status.csv \ + > $entry/status.part.html + done +} + +results-html-one() { + local csv_in=$1 + echo "$csv_in -> HTML" + + # .../raw/Settings.HomePage2/2015-03-01/results.csv -> + # .../cooked/Settings.HomePage2/2015-03-01.part.html + # (This saves some directories) + local html_out=$(echo $csv_in | sed -e 's|/raw/|/cooked/|; s|/results.csv|.part.html|') + + TOOLS-csv-to-html < $csv_in > $html_out +} + +results-html() { + local job_dir=${1:-_tmp/results-10} + + find $job_dir -name results.csv \ + | xargs -n 1 --verbose --no-run-if-empty -- $0 results-html-one +} + +# Build parts of the HTML +build-html1() { + local job_dir=${1:-_tmp/results-10} + + symlink-static dist $job_dir + + # writes overview.part.html, which is loaded by overview.html + overview-part-html $job_dir + + # Writes status.part.html for each metric + metric-part-html $job_dir +} + +# +# Association Analysis +# + +readonly ASSOC_TEST_JOB_DIR=~/rappor/chrome-assoc-smoke/smoke5-assoc + +# Write HTML fragment based on CSV. +assoc-overview-part-html() { + local job_dir=${1:-$ASSOC_TEST_JOB_DIR} + local html_path=$job_dir/cooked/assoc-overview.part.html + + # Sort by descending date! + + TOOLS-csv-to-html \ + --col-format 'metric {metric}' \ + < $job_dir/cooked/assoc-overview.csv \ + > $html_path + echo "Wrote $html_path" +} + +assoc-metric-part-html-one() { + local csv_path=$1 + local html_path=$(echo $csv_path | sed 's/.csv$/.part.html/') + + local metric_dir=$(dirname $csv_path) + local metric_name=$(basename $metric_dir) # e.g. interstitial.harmful + + local fmt='days {days}' + + TOOLS-csv-to-html \ + --def "metric $metric_name" \ + --col-format "$fmt" \ + < $csv_path \ + > $html_path + + echo "Wrote $html_path" +} + +assoc-metric-part-html() { + local job_dir=${1:-$ASSOC_TEST_JOB_DIR} + # Testing it out. This should probably be a different dir. + + find $job_dir/cooked -name metric-status.csv \ + | xargs -n 1 --verbose --no-run-if-empty -- $0 assoc-metric-part-html-one +} + +# TODO: +# - Construct link in JavaScript instead? It has more information. The +# pair-metadata.txt file is a hack. + +assoc-pair-part-html-one() { + local csv_path=$1 + local html_path=$(echo $csv_path | sed 's/.csv$/.part.html/') + + local pair_dir_path=$(dirname $csv_path) + local pair_dir_name=$(basename $pair_dir_path) # e.g. domain_X_flags_IS_REPEAT_VISIT + + # This file is generated by metric_status.R for each pair of variables. + local metadata="$pair_dir_path/pair-metadata.txt" + # Read one variable per line. + { read metric_name; read var1; read var2; } < $metadata + + local fmt1='date {date}' + local fmt2="status {status}" + + TOOLS-csv-to-html \ + --def "metric $metric_name" \ + --def "var1 $var1" \ + --def "var2 $var2" \ + --col-format "$fmt1" \ + --col-format "$fmt2" \ + < $csv_path \ + > $html_path +} + +assoc-pair-part-html() { + local job_dir=${1:-~/rappor/chrome-assoc-smoke/smoke3} + # Testing it out. This should probably be a different dir. + + find $job_dir/cooked -name pair-status.csv \ + | xargs -n 1 --verbose -- $0 assoc-pair-part-html-one + + return + + # OLD STUFF + for entry in $job_dir/cooked/*; do + # Only do it for dirs + if ! test -d $entry; then + continue + fi + # Now it's a metric dir + echo $entry + + local metric_name=$(basename $entry) + + # Convert status.csv to status.part.html (a fragment) + + # NOTE: counts path could be useful. You need the input tree though. Hash + # it? Or point to the git link. + + # Link to raw CSV + #--col-format 'date {date}' \ + + # TODO: Link to ui/results_viewer.html#{metric}_{date} + # And that needs some JavaScript to load the correct fragment. + # I guess you could do the same with metric.html. Currently it uses a + # symlink. + + # Before job ID: + # --col-format 'date {date}' \ + # --col-format 'status {status}' \ + + local fmt1='date {date}' + local fmt2='status {status}' + + TOOLS-csv-to-html \ + --def "metric $metric_name" \ + --col-format "$fmt1" \ + --col-format "$fmt2" \ + < $entry/status.csv \ + > $entry/status.part.html + done +} + +assoc-day-part-html-one() { + local csv_in=$1 + echo "$csv_in -> HTML" + + # .../raw/interstitial.harmful/a_X_b/2015-03-01/assoc-results.csv -> + # .../cooked/interstitial.harmful/a_X_b/2015-03-01.part.html + # (This saves some directories) + local html_out=$(echo $csv_in | sed -e 's|/raw/|/cooked/|; s|/assoc-results.csv|.part.html|') + + TOOLS-csv-to-html --as-percent proportion < $csv_in > $html_out +} + +assoc-day-part-html() { + local job_dir=${1:-_tmp/results-10} + + find $job_dir -name assoc-results.csv \ + | xargs -n 1 --verbose --no-run-if-empty -- $0 assoc-day-part-html-one +} + +lint-html() { + set -o xtrace + set +o errexit # don't fail fast + tidy -errors -quiet ui/metric.html + tidy -errors -quiet ui/overview.html + tidy -errors -quiet ui/histograms.html +} + +# Directory we should serve from +readonly WWW_DIR=_tmp + +serve() { + local port=${1:-7999} + cd $WWW_DIR && python -m SimpleHTTPServer $port +} + +"$@" diff --git a/pipeline/util.py b/pipeline/util.py new file mode 100644 index 00000000..c5174834 --- /dev/null +++ b/pipeline/util.py @@ -0,0 +1,9 @@ +"""Common functions.""" + +import sys + + +def log(msg, *args): + if args: + msg = msg % args + print >>sys.stderr, msg diff --git a/third_party/dygraph-combined.js b/third_party/dygraph-combined.js new file mode 100644 index 00000000..7d6121e1 --- /dev/null +++ b/third_party/dygraph-combined.js @@ -0,0 +1,6 @@ +/*! @license Copyright 2014 Dan Vanderkam (danvdk@gmail.com) MIT-licensed (http://opensource.org/licenses/MIT) */ +!function(t){"use strict";for(var e,a,i={},r=function(){},n="memory".split(","),o="assert,clear,count,debug,dir,dirxml,error,exception,group,groupCollapsed,groupEnd,info,log,markTimeline,profile,profiles,profileEnd,show,table,time,timeEnd,timeline,timelineEnd,timeStamp,trace,warn".split(",");e=n.pop();)t[e]=t[e]||i;for(;a=o.pop();)t[a]=t[a]||r}(this.console=this.console||{}),function(){"use strict";CanvasRenderingContext2D.prototype.installPattern=function(t){if("undefined"!=typeof this.isPatternInstalled)throw"Must un-install old line pattern before installing a new one.";this.isPatternInstalled=!0;var e=[0,0],a=[],i=this.beginPath,r=this.lineTo,n=this.moveTo,o=this.stroke;this.uninstallPattern=function(){this.beginPath=i,this.lineTo=r,this.moveTo=n,this.stroke=o,this.uninstallPattern=void 0,this.isPatternInstalled=void 0},this.beginPath=function(){a=[],i.call(this)},this.moveTo=function(t,e){a.push([[t,e]]),n.call(this,t,e)},this.lineTo=function(t,e){var i=a[a.length-1];i.push([t,e])},this.stroke=function(){if(0===a.length)return void o.call(this);for(var i=0;if;){var x=t[v];f+=e[1]?e[1]:x,f>y?(e=[v,f-y],f=y):e=[(v+1)%t.length,0],v%2===0?r.call(this,f,0):n.call(this,f,0),v=(v+1)%t.length}this.restore(),l=g,h=d}o.call(this),a=[]}},CanvasRenderingContext2D.prototype.uninstallPattern=function(){throw"Must install a line pattern before uninstalling it."}}();var DygraphOptions=function(){return function(){"use strict";var t=function(t){this.dygraph_=t,this.yAxes_=[],this.xAxis_={},this.series_={},this.global_=this.dygraph_.attrs_,this.user_=this.dygraph_.user_attrs_||{},this.labels_=[],this.highlightSeries_=this.get("highlightSeriesOpts")||{},this.reparseSeries()};t.AXIS_STRING_MAPPINGS_={y:0,Y:0,y1:0,Y1:0,y2:1,Y2:1},t.axisToIndex_=function(e){if("string"==typeof e){if(t.AXIS_STRING_MAPPINGS_.hasOwnProperty(e))return t.AXIS_STRING_MAPPINGS_[e];throw"Unknown axis : "+e}if("number"==typeof e){if(0===e||1===e)return e;throw"Dygraphs only supports two y-axes, indexed from 0-1."}if(e)throw"Unknown axis : "+e;return 0},t.prototype.reparseSeries=function(){var e=this.get("labels");if(e){this.labels_=e.slice(1),this.yAxes_=[{series:[],options:{}}],this.xAxis_={options:{}},this.series_={};var a=!this.user_.series;if(a){for(var i=0,r=0;r1&&Dygraph.update(this.yAxes_[1].options,h.y2||{}),Dygraph.update(this.xAxis_.options,h.x||{})}},t.prototype.get=function(t){var e=this.getGlobalUser_(t);return null!==e?e:this.getGlobalDefault_(t)},t.prototype.getGlobalUser_=function(t){return this.user_.hasOwnProperty(t)?this.user_[t]:null},t.prototype.getGlobalDefault_=function(t){return this.global_.hasOwnProperty(t)?this.global_[t]:Dygraph.DEFAULT_ATTRS.hasOwnProperty(t)?Dygraph.DEFAULT_ATTRS[t]:null},t.prototype.getForAxis=function(t,e){var a,i;if("number"==typeof e)a=e,i=0===a?"y":"y2";else{if("y1"==e&&(e="y"),"y"==e)a=0;else if("y2"==e)a=1;else{if("x"!=e)throw"Unknown axis "+e;a=-1}i=e}var r=-1==a?this.xAxis_:this.yAxes_[a];if(r){var n=r.options;if(n.hasOwnProperty(t))return n[t]}if("x"!==e||"logscale"!==t){var o=this.getGlobalUser_(t);if(null!==o)return o}var s=Dygraph.DEFAULT_ATTRS.axes[i];return s.hasOwnProperty(t)?s[t]:this.getGlobalDefault_(t)},t.prototype.getForSeries=function(t,e){if(e===this.dygraph_.getHighlightSeries()&&this.highlightSeries_.hasOwnProperty(t))return this.highlightSeries_[t];if(!this.series_.hasOwnProperty(e))throw"Unknown series: "+e;var a=this.series_[e],i=a.options;return i.hasOwnProperty(t)?i[t]:this.getForAxis(t,a.yAxis)},t.prototype.numAxes=function(){return this.yAxes_.length},t.prototype.axisForSeries=function(t){return this.series_[t].yAxis},t.prototype.axisOptions=function(t){return this.yAxes_[t].options},t.prototype.seriesForAxis=function(t){return this.yAxes_[t].series},t.prototype.seriesNames=function(){return this.labels_};return t}()}(),DygraphLayout=function(){"use strict";var t=function(t){this.dygraph_=t,this.points=[],this.setNames=[],this.annotations=[],this.yAxes_=null,this.xTicks_=null,this.yTicks_=null};return t.prototype.addDataset=function(t,e){this.points.push(e),this.setNames.push(t)},t.prototype.getPlotArea=function(){return this.area_},t.prototype.computePlotArea=function(){var t={x:0,y:0};t.w=this.dygraph_.width_-t.x-this.dygraph_.getOption("rightGap"),t.h=this.dygraph_.height_;var e={chart_div:this.dygraph_.graphDiv,reserveSpaceLeft:function(e){var a={x:t.x,y:t.y,w:e,h:t.h};return t.x+=e,t.w-=e,a},reserveSpaceRight:function(e){var a={x:t.x+t.w-e,y:t.y,w:e,h:t.h};return t.w-=e,a},reserveSpaceTop:function(e){var a={x:t.x,y:t.y,w:t.w,h:e};return t.y+=e,t.h-=e,a},reserveSpaceBottom:function(e){var a={x:t.x,y:t.y+t.h-e,w:t.w,h:e};return t.h-=e,a},chartRect:function(){return{x:t.x,y:t.y,w:t.w,h:t.h}}};this.dygraph_.cascadeEvents_("layout",e),this.area_=t},t.prototype.setAnnotations=function(t){this.annotations=[];for(var e=this.dygraph_.getOption("xValueParser")||function(t){return t},a=0;a=0&&1>i&&this.xticks.push([i,a]);for(this.yticks=[],t=0;t0&&1>=i&&this.yticks.push([t,i,a])},t.prototype._evaluateAnnotations=function(){var t,e={};for(t=0;t=0;i--)a.childNodes[i].className==e&&a.removeChild(a.childNodes[i]);for(var r=document.bgColor,n=this.dygraph_.graphDiv;n!=document;){var o=n.currentStyle.backgroundColor;if(o&&"transparent"!=o){r=o;break}n=n.parentNode}var s=this.area;t({x:0,y:0,w:s.x,h:this.height}),t({x:s.x,y:0,w:this.width-s.x,h:s.y}),t({x:s.x+s.w,y:0,w:this.width-s.x-s.w,h:this.height}),t({x:s.x,y:s.y+s.h,w:this.width-s.x,h:this.height-s.h-s.y})},t._getIteratorPredicate=function(e){return e?t._predicateThatSkipsEmptyPoints:null},t._predicateThatSkipsEmptyPoints=function(t,e){return null!==t[e].yval},t._drawStyledLine=function(e,a,i,r,n,o,s){var l=e.dygraph,h=l.getBooleanOption("stepPlot",e.setName);Dygraph.isArrayLike(r)||(r=null);var p=l.getBooleanOption("drawGapEdgePoints",e.setName),g=e.points,d=e.setName,u=Dygraph.createIterator(g,0,g.length,t._getIteratorPredicate(l.getBooleanOption("connectSeparatedPoints",d))),c=r&&r.length>=2,y=e.drawingContext;y.save(),c&&y.installPattern(r);var _=t._drawSeries(e,u,i,s,n,p,h,a);t._drawPointsOnLine(e,_,o,a,s),c&&y.uninstallPattern(),y.restore()},t._drawSeries=function(t,e,a,i,r,n,o,s){var l,h,p=null,g=null,d=null,u=[],c=!0,y=t.drawingContext;y.beginPath(),y.strokeStyle=s,y.lineWidth=a;for(var _=e.array_,v=e.end_,f=e.predicate_,x=e.start_;v>x;x++){if(h=_[x],f){for(;v>x&&!f(_,x);)x++;if(x==v)break;h=_[x]}if(null===h.canvasy||h.canvasy!=h.canvasy)o&&null!==p&&(y.moveTo(p,g),y.lineTo(h.canvasx,g)),p=g=null;else{if(l=!1,n||!p){e.nextIdx_=x,e.next(),d=e.hasNext?e.peek.canvasy:null;var m=null===d||d!=d;l=!p&&m,n&&(!c&&!p||e.hasNext&&m)&&(l=!0)}null!==p?a&&(o&&(y.moveTo(p,g),y.lineTo(h.canvasx,g)),y.lineTo(h.canvasx,h.canvasy)):y.moveTo(h.canvasx,h.canvasy),(r||l)&&u.push([h.canvasx,h.canvasy,h.idx]),p=h.canvasx,g=h.canvasy}c=!1}return y.stroke(),u},t._drawPointsOnLine=function(t,e,a,i,r){for(var n=t.drawingContext,o=0;o0;a--){var i=e[a];if(i[0]==n){var o=e[a-1];o[1]==i[1]&&o[2]==i[2]&&e.splice(a,1)}}for(var a=0;a2&&!t){var s=0;e[0][0]==n&&s++;for(var l=null,h=null,a=s;ae[h][2]&&(h=a)}}var g=e[l],d=e[h];e.splice(s,e.length-s),h>l?(e.push(g),e.push(d)):l>h?(e.push(d),e.push(g)):e.push(g)}}},l=function(a){s(a);for(var l=0,h=e.length;h>l;l++){var p=e[l];p[0]==r?t.lineTo(p[1],p[2]):p[0]==n&&t.moveTo(p[1],p[2])}e.length&&(i=e[e.length-1][1]),o+=e.length,e=[]},h=function(t,r,n){var o=Math.round(r);if(null===a||o!=a){var s=a-i>1,h=o-a>1,p=s||h;l(p),a=o}e.push([t,r,n])};return{moveTo:function(t,e){h(n,t,e)},lineTo:function(t,e){h(r,t,e)},stroke:function(){l(!0),t.stroke()},fill:function(){l(!0),t.fill()},beginPath:function(){l(!0),t.beginPath()},closePath:function(){l(!0),t.closePath()},_count:function(){return o}}},t._fillPlotter=function(e){if(!e.singleSeriesName&&0===e.seriesIndex){for(var a=e.dygraph,i=a.getLabels().slice(1),r=i.length;r>=0;r--)a.visibility()[r]||i.splice(r,1);var n=function(){for(var t=0;t=0;r--){var n=i[r];t.lineTo(n[0],n[1])}},_=p-1;_>=0;_--){var v=e.drawingContext,f=i[_];if(a.getBooleanOption("fillGraph",f)){var x=a.getBooleanOption("stepPlot",f),m=u[_],D=a.axisPropertiesForSeries(f),w=1+D.minyval*D.yscale;0>w?w=0:w>1&&(w=1),w=l.h*w+l.y;var A,b=h[_],T=Dygraph.createIterator(b,0,b.length,t._getIteratorPredicate(a.getBooleanOption("connectSeparatedPoints",f))),E=0/0,C=[-1,-1],L=Dygraph.toRGB_(m),P="rgba("+L.r+","+L.g+","+L.b+","+g+")";v.fillStyle=P,v.beginPath();var S,O=!0;(b.length>2*a.width_||Dygraph.FORCE_FAST_PROXY)&&(v=t._fastCanvasProxy(v));for(var M,R=[];T.hasNext;)if(M=T.next(),Dygraph.isOK(M.y)||x){if(d){if(!O&&S==M.xval)continue;O=!1,S=M.xval,o=c[M.canvasx];var F;F=void 0===o?w:s?o[0]:o,A=[M.canvasy,F],x?-1===C[0]?c[M.canvasx]=[M.canvasy,w]:c[M.canvasx]=[M.canvasy,C[0]]:c[M.canvasx]=M.canvasy}else A=isNaN(M.canvasy)&&x?[l.y+l.h,w]:[M.canvasy,w];isNaN(E)?(v.moveTo(M.canvasx,A[1]),v.lineTo(M.canvasx,A[0])):(x?(v.lineTo(M.canvasx,C[0]),v.lineTo(M.canvasx,A[0])):v.lineTo(M.canvasx,A[0]),d&&(R.push([E,C[1]]),R.push(s&&o?[M.canvasx,o[1]]:[M.canvasx,A[1]]))),C=A,E=M.canvasx}else y(v,E,C[1],R),R=[],E=0/0,null===M.y_stacked||isNaN(M.y_stacked)||(c[M.canvasx]=l.h*M.y_stacked+l.y);s=x,A&&M&&(y(v,M.canvasx,A[1],R),R=[]),v.fill()}}}},t}(),Dygraph=function(){"use strict";var t=function(t,e,a,i){this.is_initial_draw_=!0,this.readyFns_=[],void 0!==i?(console.warn("Using deprecated four-argument dygraph constructor"),this.__old_init__(t,e,a,i)):this.__init__(t,e,a)};return t.NAME="Dygraph",t.VERSION="1.1.1",t.__repr__=function(){return"["+t.NAME+" "+t.VERSION+"]"},t.toString=function(){return t.__repr__()},t.DEFAULT_ROLL_PERIOD=1,t.DEFAULT_WIDTH=480,t.DEFAULT_HEIGHT=320,t.ANIMATION_STEPS=12,t.ANIMATION_DURATION=200,t.KMB_LABELS=["K","M","B","T","Q"],t.KMG2_BIG_LABELS=["k","M","G","T","P","E","Z","Y"],t.KMG2_SMALL_LABELS=["m","u","n","p","f","a","z","y"],t.numberValueFormatter=function(e,a){var i=a("sigFigs");if(null!==i)return t.floatFormat(e,i);var r,n=a("digitsAfterDecimal"),o=a("maxNumberWidth"),s=a("labelsKMB"),l=a("labelsKMG2");if(r=0!==e&&(Math.abs(e)>=Math.pow(10,o)||Math.abs(e)=0;c--,u/=h)if(d>=u){r=t.round_(e/u,n)+p[c];break}if(l){var y=String(e.toExponential()).split("e-");2===y.length&&y[1]>=3&&y[1]<=24&&(r=y[1]%3>0?t.round_(y[0]/t.pow(10,y[1]%3),n):Number(y[0]).toFixed(2),r+=g[Math.floor(y[1]/3)-1])}}return r},t.numberAxisLabelFormatter=function(e,a,i){return t.numberValueFormatter.call(this,e,i)},t.SHORT_MONTH_NAMES_=["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],t.dateAxisLabelFormatter=function(e,a,i){var r=i("labelsUTC"),n=r?t.DateAccessorsUTC:t.DateAccessorsLocal,o=n.getFullYear(e),s=n.getMonth(e),l=n.getDate(e),h=n.getHours(e),p=n.getMinutes(e),g=n.getSeconds(e),d=n.getSeconds(e);if(a>=t.DECADAL)return""+o;if(a>=t.MONTHLY)return t.SHORT_MONTH_NAMES_[s]+" "+o;var u=3600*h+60*p+g+.001*d;return 0===u||a>=t.DAILY?t.zeropad(l)+" "+t.SHORT_MONTH_NAMES_[s]:t.hmsString_(h,p,g)},t.dateAxisFormatter=t.dateAxisLabelFormatter,t.dateValueFormatter=function(e,a){return t.dateString_(e,a("labelsUTC"))},t.Plotters=DygraphCanvasRenderer._Plotters,t.DEFAULT_ATTRS={highlightCircleSize:3,highlightSeriesOpts:null,highlightSeriesBackgroundAlpha:.5,labelsDivWidth:250,labelsDivStyles:{},labelsSeparateLines:!1,labelsShowZeroValues:!0,labelsKMB:!1,labelsKMG2:!1,showLabelsOnHighlight:!0,digitsAfterDecimal:2,maxNumberWidth:6,sigFigs:null,strokeWidth:1,strokeBorderWidth:0,strokeBorderColor:"white",axisTickSize:3,axisLabelFontSize:14,rightGap:5,showRoller:!1,xValueParser:t.dateParser,delimiter:",",sigma:2,errorBars:!1,fractions:!1,wilsonInterval:!0,customBars:!1,fillGraph:!1,fillAlpha:.15,connectSeparatedPoints:!1,stackedGraph:!1,stackedGraphNaNFill:"all",hideOverlayOnMouseOut:!0,legend:"onmouseover",stepPlot:!1,avoidMinZero:!1,xRangePad:0,yRangePad:null,drawAxesAtZero:!1,titleHeight:28,xLabelHeight:18,yLabelWidth:18,drawXAxis:!0,drawYAxis:!0,axisLineColor:"black",axisLineWidth:.3,gridLineWidth:.3,axisLabelColor:"black",axisLabelWidth:50,drawYGrid:!0,drawXGrid:!0,gridLineColor:"rgb(128,128,128)",interactionModel:null,animatedZooms:!1,showRangeSelector:!1,rangeSelectorHeight:40,rangeSelectorPlotStrokeColor:"#808FAB",rangeSelectorPlotFillColor:"#A7B1C4",showInRangeSelector:null,plotter:[t.Plotters.fillPlotter,t.Plotters.errorPlotter,t.Plotters.linePlotter],plugins:[],axes:{x:{pixelsPerLabel:70,axisLabelWidth:60,axisLabelFormatter:t.dateAxisLabelFormatter,valueFormatter:t.dateValueFormatter,drawGrid:!0,drawAxis:!0,independentTicks:!0,ticker:null},y:{axisLabelWidth:50,pixelsPerLabel:30,valueFormatter:t.numberValueFormatter,axisLabelFormatter:t.numberAxisLabelFormatter,drawGrid:!0,drawAxis:!0,independentTicks:!0,ticker:null},y2:{axisLabelWidth:50,pixelsPerLabel:30,valueFormatter:t.numberValueFormatter,axisLabelFormatter:t.numberAxisLabelFormatter,drawAxis:!0,drawGrid:!1,independentTicks:!1,ticker:null}}},t.HORIZONTAL=1,t.VERTICAL=2,t.PLUGINS=[],t.addedAnnotationCSS=!1,t.prototype.__old_init__=function(e,a,i,r){if(null!==i){for(var n=["Date"],o=0;o=0;n--){var o=r[n][0],s=r[n][1];if(s.call(o,i),i.propagationStopped)break}return i.defaultPrevented},t.prototype.getPluginInstance_=function(t){for(var e=0;et||t>=this.axes_.length)return null;var e=this.axes_[t];return[e.computedValueRange[0],e.computedValueRange[1]]},t.prototype.yAxisRanges=function(){for(var t=[],e=0;et||t>this.rawData_.length?null:0>e||e>this.rawData_[t].length?null:this.rawData_[t][e]},t.prototype.createInterface_=function(){var e=this.maindiv_;this.graphDiv=document.createElement("div"),this.graphDiv.style.textAlign="left",this.graphDiv.style.position="relative",e.appendChild(this.graphDiv),this.canvas_=t.createCanvas(),this.canvas_.style.position="absolute",this.hidden_=this.createPlotKitCanvas_(this.canvas_),this.canvas_ctx_=t.getContext(this.canvas_),this.hidden_ctx_=t.getContext(this.hidden_),this.resizeElements_(),this.graphDiv.appendChild(this.hidden_),this.graphDiv.appendChild(this.canvas_),this.mouseEventElement_=this.createMouseEventElement_(),this.layout_=new DygraphLayout(this);var a=this;this.mouseMoveHandler_=function(t){a.mouseMove_(t)},this.mouseOutHandler_=function(e){var i=e.target||e.fromElement,r=e.relatedTarget||e.toElement;t.isNodeContainedBy(i,a.graphDiv)&&!t.isNodeContainedBy(r,a.graphDiv)&&a.mouseOut_(e)},this.addAndTrackEvent(window,"mouseout",this.mouseOutHandler_),this.addAndTrackEvent(this.mouseEventElement_,"mousemove",this.mouseMoveHandler_),this.resizeHandler_||(this.resizeHandler_=function(t){a.resize()},this.addAndTrackEvent(window,"resize",this.resizeHandler_))},t.prototype.resizeElements_=function(){this.graphDiv.style.width=this.width_+"px",this.graphDiv.style.height=this.height_+"px";var e=t.getContextPixelRatio(this.canvas_ctx_);this.canvas_.width=this.width_*e,this.canvas_.height=this.height_*e,this.canvas_.style.width=this.width_+"px",this.canvas_.style.height=this.height_+"px",1!==e&&this.canvas_ctx_.scale(e,e);var a=t.getContextPixelRatio(this.hidden_ctx_);this.hidden_.width=this.width_*a,this.hidden_.height=this.height_*a,this.hidden_.style.width=this.width_+"px",this.hidden_.style.height=this.height_+"px",1!==a&&this.hidden_ctx_.scale(a,a)},t.prototype.destroy=function(){this.canvas_ctx_.restore(),this.hidden_ctx_.restore();for(var e=this.plugins_.length-1;e>=0;e--){var a=this.plugins_.pop();a.plugin.destroy&&a.plugin.destroy()}var i=function(t){for(;t.hasChildNodes();)i(t.firstChild),t.removeChild(t.firstChild)};this.removeTrackedEvents_(),t.removeEvent(window,"mouseout",this.mouseOutHandler_),t.removeEvent(this.mouseEventElement_,"mousemove",this.mouseMoveHandler_),t.removeEvent(window,"resize",this.resizeHandler_),this.resizeHandler_=null,i(this.maindiv_);var r=function(t){for(var e in t)"object"==typeof t[e]&&(t[e]=null)};r(this.layout_),r(this.plotter_),r(this)},t.prototype.createPlotKitCanvas_=function(e){var a=t.createCanvas();return a.style.position="absolute",a.style.top=e.style.top,a.style.left=e.style.left,a.width=this.width_,a.height=this.height_,a.style.width=this.width_+"px",a.style.height=this.height_+"px",a},t.prototype.createMouseEventElement_=function(){if(this.isUsingExcanvas_){var t=document.createElement("div");return t.style.position="absolute",t.style.backgroundColor="white",t.style.filter="alpha(opacity=0)",t.style.width=this.width_+"px",t.style.height=this.height_+"px",this.graphDiv.appendChild(t),t}return this.canvas_},t.prototype.setColors_=function(){var e=this.getLabels(),a=e.length-1;this.colors_=[],this.colorsMap_={};for(var i=this.getNumericOption("colorSaturation")||1,r=this.getNumericOption("colorValue")||.5,n=Math.ceil(a/2),o=this.getOption("colors"),s=this.visibility(),l=0;a>l;l++)if(s[l]){ +var h=e[l+1],p=this.attributes_.getForSeries("color",h);if(!p)if(o)p=o[l%o.length];else{var g=l%2?n+(l+1)/2:Math.ceil((l+1)/2),d=1*g/(1+a);p=t.hsvToRGB(d,i,r)}this.colors_.push(p),this.colorsMap_[h]=p}},t.prototype.getColors=function(){return this.colors_},t.prototype.getPropertiesForSeries=function(t){for(var e=-1,a=this.getLabels(),i=1;i=o;o++)s=t.zoomAnimationFunction(o,l),h[o-1]=[e[0]*(1-s)+s*a[0],e[1]*(1-s)+s*a[1]];if(null!==i&&null!==r)for(o=1;l>=o;o++){s=t.zoomAnimationFunction(o,l);for(var g=[],d=0;dl;l++){var h=o[l];if(t.isValidPoint(h,!0)){var p=Math.abs(h.canvasx-e);a>p&&(a=p,i=h.idx)}}return i},t.prototype.findClosestPoint=function(e,a){for(var i,r,n,o,s,l,h,p=1/0,g=this.layout_.points.length-1;g>=0;--g)for(var d=this.layout_.points[g],u=0;ui&&(p=i,s=o,l=g,h=o.idx));var c=this.layout_.setNames[l];return{row:h,seriesName:c,point:s}},t.prototype.findStackedPoint=function(e,a){for(var i,r,n=this.findClosestRow(e),o=0;o=h.length)){var p=h[l];if(t.isValidPoint(p)){var g=p.canvasy;if(e>p.canvasx&&l+10){var c=(e-p.canvasx)/u;g+=c*(d.canvasy-p.canvasy)}}}else if(e0){var y=h[l-1];if(t.isValidPoint(y)){var u=p.canvasx-y.canvasx;if(u>0){var c=(p.canvasx-e)/u;g+=c*(y.canvasy-p.canvasy)}}}(0===o||a>g)&&(i=p,r=o)}}}var _=this.layout_.setNames[r];return{row:n,seriesName:_,point:i}},t.prototype.mouseMove_=function(t){var e=this.layout_.points;if(void 0!==e&&null!==e){var a=this.eventToDomCoords(t),i=a[0],r=a[1],n=this.getOption("highlightSeriesOpts"),o=!1;if(n&&!this.isSeriesLocked()){var s;s=this.getBooleanOption("stackedGraph")?this.findStackedPoint(i,r):this.findClosestPoint(i,r),o=this.setSelection(s.row,s.seriesName)}else{var l=this.findClosestRow(i);o=this.setSelection(l)}var h=this.getFunctionOption("highlightCallback");h&&o&&h.call(this,t,this.lastx_,this.selPoints_,this.lastRow_,this.highlightSet_)}},t.prototype.getLeftBoundary_=function(t){if(this.boundaryIds_[t])return this.boundaryIds_[t][0];for(var e=0;ee?r:a-r;if(0>=n)return void(this.fadeLevel&&this.updateSelection_(1));var o=++this.animateId,s=this;t.repeatAndCleanup(function(t){s.animateId==o&&(s.fadeLevel+=e,0===s.fadeLevel?s.clearSelection():s.updateSelection_(s.fadeLevel/a))},n,i,function(){})},t.prototype.updateSelection_=function(e){this.cascadeEvents_("select",{selectedRow:this.lastRow_,selectedX:this.lastx_,selectedPoints:this.selPoints_});var a,i=this.canvas_ctx_;if(this.getOption("highlightSeriesOpts")){i.clearRect(0,0,this.width_,this.height_);var r=1-this.getNumericOption("highlightSeriesBackgroundAlpha");if(r){var n=!0;if(n){if(void 0===e)return void this.animateSelection_(1);r*=e}i.fillStyle="rgba(255,255,255,"+r+")",i.fillRect(0,0,this.width_,this.height_)}this.plotter_._renderLineChart(this.highlightSet_,i)}else if(this.previousVerticalX_>=0){var o=0,s=this.attr_("labels");for(a=1;ao&&(o=l)}var h=this.previousVerticalX_;i.clearRect(h-o-1,0,2*o+2,this.height_)}if(this.isUsingExcanvas_&&this.currentZoomRectArgs_&&t.prototype.drawZoomRect_.apply(this,this.currentZoomRectArgs_),this.selPoints_.length>0){var p=this.selPoints_[0].canvasx;for(i.save(),a=0;a=0){t!=this.lastRow_&&(i=!0),this.lastRow_=t;for(var r=0;r=0&&(i=!0),this.lastRow_=-1;return this.selPoints_.length?this.lastx_=this.selPoints_[0].xval:this.lastx_=-1,void 0!==e&&(this.highlightSet_!==e&&(i=!0),this.highlightSet_=e),void 0!==a&&(this.lockedSet_=a),i&&this.updateSelection_(void 0),i},t.prototype.mouseOut_=function(t){this.getFunctionOption("unhighlightCallback")&&this.getFunctionOption("unhighlightCallback").call(this,t),this.getBooleanOption("hideOverlayOnMouseOut")&&!this.lockedSet_&&this.clearSelection()},t.prototype.clearSelection=function(){return this.cascadeEvents_("deselect",{}),this.lockedSet_=!1,this.fadeLevel?void this.animateSelection_(-1):(this.canvas_ctx_.clearRect(0,0,this.width_,this.height_),this.fadeLevel=0,this.selPoints_=[],this.lastx_=-1,this.lastRow_=-1,void(this.highlightSet_=null))},t.prototype.getSelection=function(){if(!this.selPoints_||this.selPoints_.length<1)return-1;for(var t=0;t1&&(a=this.dataHandler_.rollingAverage(a,this.rollPeriod_,this.attributes_)),this.rolledSeries_.push(a)}this.drawGraph_();var i=new Date;this.drawingTimeMs_=i-t},t.PointType=void 0,t.stackPoints_=function(t,e,a,i){for(var r=null,n=null,o=null,s=-1,l=function(e){if(!(s>=e))for(var a=e;aa[1]&&(a[1]=u),u=1;i--)if(this.visibility()[i-1]){if(a){l=e[i];var c=a[0],y=a[1];for(n=null,o=null,r=0;r=c&&null===n&&(n=r),l[r][0]<=y&&(o=r);null===n&&(n=0);for(var _=n,v=!0;v&&_>0;)_--,v=null===l[_][1];null===o&&(o=l.length-1);var f=o;for(v=!0;v&&f0&&(this.setIndexByName_[n[0]]=0);for(var o=0,s=1;s0;){var a=this.readyFns_.pop();a(this)}},t.prototype.computeYAxes_=function(){var e,a,i,r,n;if(void 0!==this.axes_&&this.user_attrs_.hasOwnProperty("valueRange")===!1)for(e=[],i=0;ii;i++)this.axes_[i].valueWindow=e[i]}for(a=0;al;l++){var h=this.axes_[l],p=this.attributes_.getForAxis("logscale",l),g=this.attributes_.getForAxis("includeZero",l),d=this.attributes_.getForAxis("independentTicks",l);if(i=this.attributes_.seriesForAxis(l),e=!0,r=.1,null!==this.getNumericOption("yRangePad")&&(e=!1,r=this.getNumericOption("yRangePad")/this.plotter_.area.h),0===i.length)h.extremeRange=[0,1];else{for(var u,c,y=1/0,_=-(1/0),v=0;v0&&(y=0),0>_&&(_=0)),y==1/0&&(y=0),_==-(1/0)&&(_=1),a=_-y,0===a&&(0!==_?a=Math.abs(_):(_=1,a=1));var f,x;if(p)if(e)f=_+r*a,x=y;else{var m=Math.exp(Math.log(a)*r);f=_*m,x=y/m}else f=_+r*a,x=y-r*a,e&&!this.getBooleanOption("avoidMinZero")&&(0>x&&y>=0&&(x=0),f>0&&0>=_&&(f=0));h.extremeRange=[x,f]}if(h.valueWindow)h.computedValueRange=[h.valueWindow[0],h.valueWindow[1]];else if(h.valueRange){var D=o(h.valueRange[0])?h.extremeRange[0]:h.valueRange[0],w=o(h.valueRange[1])?h.extremeRange[1]:h.valueRange[1];if(!e)if(h.logscale){var m=Math.exp(Math.log(a)*r);D*=m,w/=m}else a=w-D,D-=a*r,w+=a*r;h.computedValueRange=[D,w]}else h.computedValueRange=h.extremeRange;if(d){h.independentTicks=d;var A=this.optionsViewForAxis_("y"+(l?"2":"")),b=A("ticker");h.ticks=b(h.computedValueRange[0],h.computedValueRange[1],this.plotter_.area.h,A,this),n||(n=h)}}if(void 0===n)throw'Configuration Error: At least one axis has to have the "independentTicks" option activated.';for(var l=0;s>l;l++){var h=this.axes_[l];if(!h.independentTicks){for(var A=this.optionsViewForAxis_("y"+(l?"2":"")),b=A("ticker"),T=n.ticks,E=n.computedValueRange[1]-n.computedValueRange[0],C=h.computedValueRange[1]-h.computedValueRange[0],L=[],P=0;P0&&"e"!=t[a-1]&&"E"!=t[a-1]||t.indexOf("/")>=0||isNaN(parseFloat(t))?e=!0:8==t.length&&t>"19700101"&&"20371231">t&&(e=!0),this.setXAxisOptions_(e)},t.prototype.setXAxisOptions_=function(e){e?(this.attrs_.xValueParser=t.dateParser,this.attrs_.axes.x.valueFormatter=t.dateValueFormatter,this.attrs_.axes.x.ticker=t.dateTicker,this.attrs_.axes.x.axisLabelFormatter=t.dateAxisLabelFormatter):(this.attrs_.xValueParser=function(t){return parseFloat(t)},this.attrs_.axes.x.valueFormatter=function(t){return t},this.attrs_.axes.x.ticker=t.numericTicks,this.attrs_.axes.x.axisLabelFormatter=this.attrs_.axes.x.valueFormatter)},t.prototype.parseCSV_=function(e){var a,i,r=[],n=t.detectLineDelimiter(e),o=e.split(n||"\n"),s=this.getStringOption("delimiter");-1==o[0].indexOf(s)&&o[0].indexOf(" ")>=0&&(s=" ");var l=0;"labels"in this.user_attrs_||(l=1,this.attrs_.labels=o[0].split(s),this.attributes_.reparseSeries());for(var h,p=0,g=!1,d=this.attr_("labels").length,u=!1,c=l;c0&&v[0]0;)e=String.fromCharCode(65+(t-1)%26)+e.toLowerCase(),t=Math.floor((t-1)/26);return e},i=e.getNumberOfColumns(),r=e.getNumberOfRows(),n=e.getColumnType(0);if("date"==n||"datetime"==n)this.attrs_.xValueParser=t.dateParser,this.attrs_.axes.x.valueFormatter=t.dateValueFormatter,this.attrs_.axes.x.ticker=t.dateTicker,this.attrs_.axes.x.axisLabelFormatter=t.dateAxisLabelFormatter;else{if("number"!=n)return console.error("only 'date', 'datetime' and 'number' types are supported for column 1 of DataTable input (Got '"+n+"')"),null;this.attrs_.xValueParser=function(t){return parseFloat(t)},this.attrs_.axes.x.valueFormatter=function(t){return t},this.attrs_.axes.x.ticker=t.numericTicks,this.attrs_.axes.x.axisLabelFormatter=this.attrs_.axes.x.valueFormatter}var o,s,l=[],h={},p=!1;for(o=1;i>o;o++){var g=e.getColumnType(o);if("number"==g)l.push(o);else if("string"==g&&this.getBooleanOption("displayAnnotations")){var d=l[l.length-1];h.hasOwnProperty(d)?h[d].push(o):h[d]=[o],p=!0}else console.error("Only 'number' is supported as a dependent type with Gviz. 'string' is only supported if displayAnnotations is true")}var u=[e.getColumnLabel(0)];for(o=0;oo;o++){var v=[];if("undefined"!=typeof e.getValue(o,0)&&null!==e.getValue(o,0)){if(v.push("date"==n||"datetime"==n?e.getValue(o,0).getTime():e.getValue(o,0)),this.getBooleanOption("errorBars"))for(s=0;i-1>s;s++)v.push([e.getValue(o,1+2*s),e.getValue(o,2+2*s)]);else{for(s=0;s0&&v[0]0&&this.setAnnotations(_,!0),this.attributes_.reparseSeries()},t.prototype.cascadeDataDidUpdateEvent_=function(){this.cascadeEvents_("dataDidUpdate",{})},t.prototype.start_=function(){var e=this.file_;if("function"==typeof e&&(e=e()),t.isArrayLike(e))this.rawData_=this.parseArray_(e),this.cascadeDataDidUpdateEvent_(),this.predraw_();else if("object"==typeof e&&"function"==typeof e.getColumnRange)this.parseDataTable_(e),this.cascadeDataDidUpdateEvent_(),this.predraw_();else if("string"==typeof e){var a=t.detectLineDelimiter(e);if(a)this.loadedEvent_(e);else{var i;i=window.XMLHttpRequest?new XMLHttpRequest:new ActiveXObject("Microsoft.XMLHTTP");var r=this;i.onreadystatechange=function(){4==i.readyState&&(200===i.status||0===i.status)&&r.loadedEvent_(i.responseText)},i.open("GET",e,!0),i.send(null)}}else console.error("Unknown data format: "+typeof e)},t.prototype.updateOptions=function(e,a){"undefined"==typeof a&&(a=!1);var i=e.file,r=t.mapLegacyOptions_(e);"rollPeriod"in r&&(this.rollPeriod_=r.rollPeriod),"dateWindow"in r&&(this.dateWindow_=r.dateWindow,"isZoomedIgnoreProgrammaticZoom"in r||(this.zoomed_x_=null!==r.dateWindow)),"valueRange"in r&&!("isZoomedIgnoreProgrammaticZoom"in r)&&(this.zoomed_y_=null!==r.valueRange);var n=t.isPixelChangingOptionList(this.attr_("labels"),r);t.updateDeep(this.user_attrs_,r),this.attributes_.reparseSeries(),i?(this.cascadeEvents_("dataWillUpdate",{}),this.file_=i,a||this.start_()):a||(n?this.predraw_():this.renderGraph_(!1))},t.mapLegacyOptions_=function(t){var e={};for(var a in t)t.hasOwnProperty(a)&&"file"!=a&&t.hasOwnProperty(a)&&(e[a]=t[a]);var i=function(t,a,i){e.axes||(e.axes={}),e.axes[t]||(e.axes[t]={}),e.axes[t][a]=i},r=function(a,r,n){"undefined"!=typeof t[a]&&(console.warn("Option "+a+" is deprecated. Use the "+n+" option for the "+r+" axis instead. (e.g. { axes : { "+r+" : { "+n+" : ... } } } (see http://dygraphs.com/per-axis.html for more information."),i(r,n,t[a]),delete e[a])};return r("xValueFormatter","x","valueFormatter"),r("pixelsPerXLabel","x","pixelsPerLabel"),r("xAxisLabelFormatter","x","axisLabelFormatter"),r("xTicker","x","ticker"),r("yValueFormatter","y","valueFormatter"),r("pixelsPerYLabel","y","pixelsPerLabel"),r("yAxisLabelFormatter","y","axisLabelFormatter"),r("yTicker","y","ticker"),r("drawXGrid","x","drawGrid"),r("drawXAxis","x","drawAxis"),r("drawYGrid","y","drawGrid"),r("drawYAxis","y","drawAxis"),r("xAxisLabelWidth","x","axisLabelWidth"),r("yAxisLabelWidth","y","axisLabelWidth"),e},t.prototype.resize=function(t,e){if(!this.resize_lock){this.resize_lock=!0,null===t!=(null===e)&&(console.warn("Dygraph.resize() should be called with zero parameters or two non-NULL parameters. Pretending it was zero."),t=e=null);var a=this.width_,i=this.height_;t?(this.maindiv_.style.width=t+"px",this.maindiv_.style.height=e+"px",this.width_=t,this.height_=e):(this.width_=this.maindiv_.clientWidth,this.height_=this.maindiv_.clientHeight),(a!=this.width_||i!=this.height_)&&(this.resizeElements_(),this.predraw_()),this.resize_lock=!1}},t.prototype.adjustRoll=function(t){this.rollPeriod_=t,this.predraw_()},t.prototype.visibility=function(){for(this.getOption("visibility")||(this.attrs_.visibility=[]);this.getOption("visibility").lengtht||t>=a.length?console.warn("invalid series number in setVisibility: "+t):(a[t]=e,this.predraw_())},t.prototype.size=function(){return{width:this.width_,height:this.height_}},t.prototype.setAnnotations=function(e,a){return t.addAnnotationRule(),this.annotations_=e,this.layout_?(this.layout_.setAnnotations(this.annotations_),void(a||this.predraw_())):void console.warn("Tried to setAnnotations before dygraph was ready. Try setting them in a ready() block. See dygraphs.com/tests/annotation.html")},t.prototype.annotations=function(){return this.annotations_},t.prototype.getLabels=function(){var t=this.attr_("labels");return t?t.slice():null},t.prototype.indexFromSetName=function(t){return this.setIndexByName_[t]},t.prototype.ready=function(t){this.is_initial_draw_?this.readyFns_.push(t):t.call(this,this)},t.addAnnotationRule=function(){if(!t.addedAnnotationCSS){var e="border: 1px solid black; background-color: white; text-align: center;",a=document.createElement("style");a.type="text/css",document.getElementsByTagName("head")[0].appendChild(a);for(var i=0;it?"0"+t:""+t},Dygraph.DateAccessorsLocal={getFullYear:function(t){return t.getFullYear()},getMonth:function(t){return t.getMonth()},getDate:function(t){return t.getDate()},getHours:function(t){return t.getHours()},getMinutes:function(t){return t.getMinutes()},getSeconds:function(t){return t.getSeconds()},getMilliseconds:function(t){return t.getMilliseconds()},getDay:function(t){return t.getDay()},makeDate:function(t,e,a,i,r,n,o){return new Date(t,e,a,i,r,n,o)}},Dygraph.DateAccessorsUTC={getFullYear:function(t){return t.getUTCFullYear()},getMonth:function(t){return t.getUTCMonth()},getDate:function(t){return t.getUTCDate()},getHours:function(t){return t.getUTCHours()},getMinutes:function(t){return t.getUTCMinutes()},getSeconds:function(t){return t.getUTCSeconds()},getMilliseconds:function(t){return t.getUTCMilliseconds()},getDay:function(t){return t.getUTCDay()},makeDate:function(t,e,a,i,r,n,o){return new Date(Date.UTC(t,e,a,i,r,n,o))}},Dygraph.hmsString_=function(t,e,a){var i=Dygraph.zeropad,r=i(t)+":"+i(e);return a&&(r+=":"+i(a)),r},Dygraph.dateString_=function(t,e){var a=Dygraph.zeropad,i=e?Dygraph.DateAccessorsUTC:Dygraph.DateAccessorsLocal,r=new Date(t),n=i.getFullYear(r),o=i.getMonth(r),s=i.getDate(r),l=i.getHours(r),h=i.getMinutes(r),p=i.getSeconds(r),g=""+n,d=a(o+1),u=a(s),c=3600*l+60*h+p,y=g+"/"+d+"/"+u;return c&&(y+=" "+Dygraph.hmsString_(l,h,p)),y},Dygraph.round_=function(t,e){var a=Math.pow(10,e);return Math.round(t*a)/a},Dygraph.binarySearch=function(t,e,a,i,r){if((null===i||void 0===i||null===r||void 0===r)&&(i=0,r=e.length-1),i>r)return-1;(null===a||void 0===a)&&(a=0);var n,o=function(t){return t>=0&&tt?a>0&&(n=s-1,o(n)&&e[n]l?0>a&&(n=s+1,o(n)&&e[n]>t)?s:Dygraph.binarySearch(t,e,a,s+1,r):-1},Dygraph.dateParser=function(t){var e,a;if((-1==t.search("-")||-1!=t.search("T")||-1!=t.search("Z"))&&(a=Dygraph.dateStrToMillis(t),a&&!isNaN(a)))return a;if(-1!=t.search("-")){for(e=t.replace("-","/","g");-1!=e.search("-");)e=e.replace("-","/");a=Dygraph.dateStrToMillis(e)}else 8==t.length?(e=t.substr(0,4)+"/"+t.substr(4,2)+"/"+t.substr(6,2),a=Dygraph.dateStrToMillis(e)):a=Dygraph.dateStrToMillis(t);return(!a||isNaN(a))&&console.error("Couldn't parse "+t+" as a date"),a},Dygraph.dateStrToMillis=function(t){return new Date(t).getTime()},Dygraph.update=function(t,e){if("undefined"!=typeof e&&null!==e)for(var a in e)e.hasOwnProperty(a)&&(t[a]=e[a]);return t},Dygraph.updateDeep=function(t,e){function a(t){return"object"==typeof Node?t instanceof Node:"object"==typeof t&&"number"==typeof t.nodeType&&"string"==typeof t.nodeName}if("undefined"!=typeof e&&null!==e)for(var i in e)e.hasOwnProperty(i)&&(null===e[i]?t[i]=null:Dygraph.isArrayLike(e[i])?t[i]=e[i].slice():a(e[i])?t[i]=e[i]:"object"==typeof e[i]?(("object"!=typeof t[i]||null===t[i])&&(t[i]={}),Dygraph.updateDeep(t[i],e[i])):t[i]=e[i]);return t},Dygraph.isArrayLike=function(t){var e=typeof t;return"object"!=e&&("function"!=e||"function"!=typeof t.item)||null===t||"number"!=typeof t.length||3===t.nodeType?!1:!0},Dygraph.isDateLike=function(t){return"object"!=typeof t||null===t||"function"!=typeof t.getTime?!1:!0},Dygraph.clone=function(t){for(var e=[],a=0;a=e||Dygraph.requestAnimFrame.call(window,function(){var e=(new Date).getTime(),h=e-o;r=n,n=Math.floor(h/a);var p=n-r,g=n+p>s;g||n>=s?(t(s),i()):(0!==p&&t(n),l())})}()};var e={annotationClickHandler:!0,annotationDblClickHandler:!0,annotationMouseOutHandler:!0,annotationMouseOverHandler:!0,axisLabelColor:!0,axisLineColor:!0,axisLineWidth:!0,clickCallback:!0,drawCallback:!0,drawHighlightPointCallback:!0,drawPoints:!0,drawPointCallback:!0,drawXGrid:!0,drawYGrid:!0,fillAlpha:!0,gridLineColor:!0,gridLineWidth:!0,hideOverlayOnMouseOut:!0,highlightCallback:!0,highlightCircleSize:!0,interactionModel:!0,isZoomedIgnoreProgrammaticZoom:!0,labelsDiv:!0,labelsDivStyles:!0,labelsDivWidth:!0,labelsKMB:!0,labelsKMG2:!0,labelsSeparateLines:!0,labelsShowZeroValues:!0,legend:!0,panEdgeFraction:!0,pixelsPerYLabel:!0,pointClickCallback:!0,pointSize:!0,rangeSelectorPlotFillColor:!0,rangeSelectorPlotStrokeColor:!0,showLabelsOnHighlight:!0,showRoller:!0,strokeWidth:!0,underlayCallback:!0,unhighlightCallback:!0,zoomCallback:!0};Dygraph.isPixelChangingOptionList=function(t,a){var i={};if(t)for(var r=1;re?1/Math.pow(t,-e):Math.pow(t,e)};var a=/^rgba?\((\d{1,3}),\s*(\d{1,3}),\s*(\d{1,3})(?:,\s*([01](?:\.\d+)?))?\)$/;Dygraph.toRGB_=function(e){var a=t(e);if(a)return a;var i=document.createElement("div");i.style.backgroundColor=e,i.style.visibility="hidden",document.body.appendChild(i);var r;return r=window.getComputedStyle?window.getComputedStyle(i,null).backgroundColor:i.currentStyle.backgroundColor,document.body.removeChild(i),t(r)},Dygraph.isCanvasSupported=function(t){var e;try{e=t||document.createElement("canvas"),e.getContext("2d")}catch(a){var i=navigator.appVersion.match(/MSIE (\d\.\d)/),r=-1!=navigator.userAgent.toLowerCase().indexOf("opera");return!i||i[1]<6||r?!1:!0}return!0},Dygraph.parseFloat_=function(t,e,a){var i=parseFloat(t);if(!isNaN(i))return i;if(/^ *$/.test(t))return null;if(/^ *nan *$/i.test(t))return 0/0;var r="Unable to parse '"+t+"' as a number";return void 0!==a&&void 0!==e&&(r+=" on line "+(1+(e||0))+" ('"+a+"') of CSV."),console.error(r),null}}(),function(){"use strict";Dygraph.GVizChart=function(t){this.container=t},Dygraph.GVizChart.prototype.draw=function(t,e){this.container.innerHTML="","undefined"!=typeof this.date_graph&&this.date_graph.destroy(),this.date_graph=new Dygraph(this.container,t,e)},Dygraph.GVizChart.prototype.setSelection=function(t){var e=!1;t.length&&(e=t[0].row),this.date_graph.setSelection(e)},Dygraph.GVizChart.prototype.getSelection=function(){var t=[],e=this.date_graph.getSelection();if(0>e)return t;for(var a=this.date_graph.layout_.points,i=0;ii&&2>r&&void 0!==e.lastx_&&-1!=e.lastx_&&Dygraph.Interaction.treatMouseOpAsClick(e,t,a),a.regionWidth=i,a.regionHeight=r},Dygraph.Interaction.startPan=function(t,e,a){var i,r;a.isPanning=!0;var n=e.xAxisRange();if(e.getOptionForAxis("logscale","x")?(a.initialLeftmostDate=Dygraph.log10(n[0]),a.dateRange=Dygraph.log10(n[1])-Dygraph.log10(n[0])):(a.initialLeftmostDate=n[0],a.dateRange=n[1]-n[0]),a.xUnitsPerPixel=a.dateRange/(e.plotter_.area.w-1),e.getNumericOption("panEdgeFraction")){var o=e.width_*e.getNumericOption("panEdgeFraction"),s=e.xAxisExtremes(),l=e.toDomXCoord(s[0])-o,h=e.toDomXCoord(s[1])+o,p=e.toDataXCoord(l),g=e.toDataXCoord(h);a.boundedDates=[p,g];var d=[],u=e.height_*e.getNumericOption("panEdgeFraction");for(i=0;ia.boundedDates[1]&&(i-=r-a.boundedDates[1],r=i+a.dateRange),e.getOptionForAxis("logscale","x")?e.dateWindow_=[Math.pow(Dygraph.LOG_SCALE,i),Math.pow(Dygraph.LOG_SCALE,r)]:e.dateWindow_=[i,r],a.is2DPan)for(var n=a.dragEndY-a.dragStartY,o=0;oi?Dygraph.VERTICAL:Dygraph.HORIZONTAL,e.drawZoomRect_(a.dragDirection,a.dragStartX,a.dragEndX,a.dragStartY,a.dragEndY,a.prevDragDirection,a.prevEndX,a.prevEndY),a.prevEndX=a.dragEndX,a.prevEndY=a.dragEndY,a.prevDragDirection=a.dragDirection},Dygraph.Interaction.treatMouseOpAsClick=function(t,e,a){for(var i=t.getFunctionOption("clickCallback"),r=t.getFunctionOption("pointClickCallback"),n=null,o=-1,s=Number.MAX_VALUE,l=0;lp)&&(s=p,o=l)}var g=t.getNumericOption("highlightCircleSize")+2;if(g*g>=s&&(n=t.selPoints_[o]),n){var d={cancelable:!0,point:n,canvasx:a.dragEndX,canvasy:a.dragEndY},u=t.cascadeEvents_("pointClick",d);if(u)return;r&&r.call(t,e,n)}var d={cancelable:!0,xval:t.lastx_,pts:t.selPoints_,canvasx:a.dragEndX,canvasy:a.dragEndY};t.cascadeEvents_("click",d)||i&&i.call(t,e,t.lastx_,t.selPoints_)},Dygraph.Interaction.endZoom=function(t,e,a){e.clearZoomRect_(),a.isZooming=!1,Dygraph.Interaction.maybeTreatMouseOpAsClick(t,e,a);var i=e.getArea();if(a.regionWidth>=10&&a.dragDirection==Dygraph.HORIZONTAL){var r=Math.min(a.dragStartX,a.dragEndX),n=Math.max(a.dragStartX,a.dragEndX);r=Math.max(r,i.x),n=Math.min(n,i.x+i.w),n>r&&e.doZoomX_(r,n),a.cancelNextDblclick=!0}else if(a.regionHeight>=10&&a.dragDirection==Dygraph.VERTICAL){var o=Math.min(a.dragStartY,a.dragEndY),s=Math.max(a.dragStartY,a.dragEndY);o=Math.max(o,i.y),s=Math.min(s,i.y+i.h),s>o&&e.doZoomY_(o,s),a.cancelNextDblclick=!0}a.dragStartX=null,a.dragStartY=null},Dygraph.Interaction.startTouch=function(t,e,a){t.preventDefault(),t.touches.length>1&&(a.startTimeForDoubleTapMs=null);for(var i=[],r=0;r=2){a.initialPinchCenter={pageX:.5*(i[0].pageX+i[1].pageX),pageY:.5*(i[0].pageY+i[1].pageY),dataX:.5*(i[0].dataX+i[1].dataX),dataY:.5*(i[0].dataY+i[1].dataY)};var o=180/Math.PI*Math.atan2(a.initialPinchCenter.pageY-i[0].pageY,i[0].pageX-a.initialPinchCenter.pageX);o=Math.abs(o),o>90&&(o=90-o),a.touchDirections={x:67.5>o,y:o>22.5}}a.initialRange={x:e.xAxisRange(),y:e.yAxisRange()}},Dygraph.Interaction.moveTouch=function(t,e,a){a.startTimeForDoubleTapMs=null;var i,r=[];for(i=0;i=2){var c=s[1].pageX-l.pageX;d=(r[1].pageX-o.pageX)/c;var y=s[1].pageY-l.pageY;u=(r[1].pageY-o.pageY)/y}d=Math.min(8,Math.max(.125,d)),u=Math.min(8,Math.max(.125,u));var _=!1;if(a.touchDirections.x&&(e.dateWindow_=[l.dataX-h.dataX+(a.initialRange.x[0]-l.dataX)/d,l.dataX-h.dataX+(a.initialRange.x[1]-l.dataX)/d],_=!0),a.touchDirections.y)for(i=0;1>i;i++){var v=e.axes_[i],f=e.attributes_.getForAxis("logscale",i);f||(v.valueWindow=[l.dataY-h.dataY+(a.initialRange.y[0]-l.dataY)/u,l.dataY-h.dataY+(a.initialRange.y[1]-l.dataY)/u],_=!0)}if(e.drawGraph_(!1),_&&r.length>1&&e.getFunctionOption("zoomCallback")){var x=e.xAxisRange();e.getFunctionOption("zoomCallback").call(e,x[0],x[1],e.yAxisRanges())}},Dygraph.Interaction.endTouch=function(t,e,a){if(0!==t.touches.length)Dygraph.Interaction.startTouch(t,e,a);else if(1==t.changedTouches.length){var i=(new Date).getTime(),r=t.changedTouches[0];a.startTimeForDoubleTapMs&&i-a.startTimeForDoubleTapMs<500&&a.doubleTapX&&Math.abs(a.doubleTapX-r.screenX)<50&&a.doubleTapY&&Math.abs(a.doubleTapY-r.screenY)<50?e.resetZoom():(a.startTimeForDoubleTapMs=i,a.doubleTapX=r.screenX,a.doubleTapY=r.screenY)}};var e=function(t,e,a){return e>t?e-t:t>a?t-a:0},a=function(t,a){var i=Dygraph.findPos(a.canvas_),r={left:i.x,right:i.x+a.canvas_.offsetWidth,top:i.y,bottom:i.y+a.canvas_.offsetHeight},n={x:Dygraph.pageX(t),y:Dygraph.pageY(t)},o=e(n.x,r.left,r.right),s=e(n.y,r.top,r.bottom);return Math.max(o,s)};Dygraph.Interaction.defaultModel={mousedown:function(e,i,r){if(!e.button||2!=e.button){r.initializeMouseDown(e,i,r),e.altKey||e.shiftKey?Dygraph.startPan(e,i,r):Dygraph.startZoom(e,i,r);var n=function(e){if(r.isZooming){var n=a(e,i);t>n?Dygraph.moveZoom(e,i,r):null!==r.dragEndX&&(r.dragEndX=null,r.dragEndY=null,i.clearZoomRect_())}else r.isPanning&&Dygraph.movePan(e,i,r)},o=function(t){r.isZooming?null!==r.dragEndX?Dygraph.endZoom(t,i,r):Dygraph.Interaction.maybeTreatMouseOpAsClick(t,i,r):r.isPanning&&Dygraph.endPan(t,i,r),Dygraph.removeEvent(document,"mousemove",n),Dygraph.removeEvent(document,"mouseup",o),r.destroy()};i.addAndTrackEvent(document,"mousemove",n),i.addAndTrackEvent(document,"mouseup",o)}},willDestroyContextMyself:!0,touchstart:function(t,e,a){Dygraph.Interaction.startTouch(t,e,a)},touchmove:function(t,e,a){Dygraph.Interaction.moveTouch(t,e,a)},touchend:function(t,e,a){Dygraph.Interaction.endTouch(t,e,a)},dblclick:function(t,e,a){if(a.cancelNextDblclick)return void(a.cancelNextDblclick=!1);var i={canvasx:a.dragEndX,canvasy:a.dragEndY};e.cascadeEvents_("dblclick",i)||t.altKey||t.shiftKey||e.resetZoom()}},Dygraph.DEFAULT_ATTRS.interactionModel=Dygraph.Interaction.defaultModel,Dygraph.defaultInteractionModel=Dygraph.Interaction.defaultModel,Dygraph.endZoom=Dygraph.Interaction.endZoom,Dygraph.moveZoom=Dygraph.Interaction.moveZoom,Dygraph.startZoom=Dygraph.Interaction.startZoom,Dygraph.endPan=Dygraph.Interaction.endPan,Dygraph.movePan=Dygraph.Interaction.movePan,Dygraph.startPan=Dygraph.Interaction.startPan,Dygraph.Interaction.nonInteractiveModel_={mousedown:function(t,e,a){a.initializeMouseDown(t,e,a)},mouseup:Dygraph.Interaction.maybeTreatMouseOpAsClick},Dygraph.Interaction.dragIsPanInteractionModel={mousedown:function(t,e,a){a.initializeMouseDown(t,e,a),Dygraph.startPan(t,e,a)},mousemove:function(t,e,a){a.isPanning&&Dygraph.movePan(t,e,a)},mouseup:function(t,e,a){a.isPanning&&Dygraph.endPan(t,e,a)}}}(),function(){"use strict";Dygraph.TickList=void 0,Dygraph.Ticker=void 0,Dygraph.numericLinearTicks=function(t,e,a,i,r,n){var o=function(t){return"logscale"===t?!1:i(t)};return Dygraph.numericTicks(t,e,a,o,r,n)},Dygraph.numericTicks=function(t,e,a,i,r,n){var o,s,l,h,p=i("pixelsPerLabel"),g=[];if(n)for(o=0;o=h/4){for(var y=u;y>=d;y--){var _=Dygraph.PREFERRED_LOG_TICK_VALUES[y],v=Math.log(_/t)/Math.log(e/t)*a,f={v:_};null===c?c={tickValue:_,pixel_coord:v}:Math.abs(v-c.pixel_coord)>=p?c={tickValue:_,pixel_coord:v}:f.label="",g.push(f)}g.reverse()}}if(0===g.length){var x,m,D=i("labelsKMG2");D?(x=[1,2,4,8,16,32,64,128,256],m=16):(x=[1,2,5,10,20,50,100],m=10);var w,A,b,T,E=Math.ceil(a/p),C=Math.abs(e-t)/E,L=Math.floor(Math.log(C)/Math.log(m)),P=Math.pow(m,L);for(s=0;sp));s++);for(A>b&&(w*=-1),o=0;h>=o;o++)l=A+o*w,g.push({v:l})}}var S=i("axisLabelFormatter");for(o=0;o=0?Dygraph.getDateAxis(t,e,o,i,r):[]},Dygraph.SECONDLY=0,Dygraph.TWO_SECONDLY=1,Dygraph.FIVE_SECONDLY=2,Dygraph.TEN_SECONDLY=3,Dygraph.THIRTY_SECONDLY=4,Dygraph.MINUTELY=5,Dygraph.TWO_MINUTELY=6,Dygraph.FIVE_MINUTELY=7,Dygraph.TEN_MINUTELY=8,Dygraph.THIRTY_MINUTELY=9,Dygraph.HOURLY=10,Dygraph.TWO_HOURLY=11,Dygraph.SIX_HOURLY=12,Dygraph.DAILY=13,Dygraph.TWO_DAILY=14,Dygraph.WEEKLY=15,Dygraph.MONTHLY=16,Dygraph.QUARTERLY=17,Dygraph.BIANNUAL=18,Dygraph.ANNUAL=19,Dygraph.DECADAL=20,Dygraph.CENTENNIAL=21,Dygraph.NUM_GRANULARITIES=22,Dygraph.DATEFIELD_Y=0,Dygraph.DATEFIELD_M=1,Dygraph.DATEFIELD_D=2,Dygraph.DATEFIELD_HH=3,Dygraph.DATEFIELD_MM=4,Dygraph.DATEFIELD_SS=5,Dygraph.DATEFIELD_MS=6,Dygraph.NUM_DATEFIELDS=7,Dygraph.TICK_PLACEMENT=[],Dygraph.TICK_PLACEMENT[Dygraph.SECONDLY]={datefield:Dygraph.DATEFIELD_SS,step:1,spacing:1e3},Dygraph.TICK_PLACEMENT[Dygraph.TWO_SECONDLY]={datefield:Dygraph.DATEFIELD_SS,step:2,spacing:2e3},Dygraph.TICK_PLACEMENT[Dygraph.FIVE_SECONDLY]={datefield:Dygraph.DATEFIELD_SS,step:5,spacing:5e3},Dygraph.TICK_PLACEMENT[Dygraph.TEN_SECONDLY]={datefield:Dygraph.DATEFIELD_SS,step:10,spacing:1e4},Dygraph.TICK_PLACEMENT[Dygraph.THIRTY_SECONDLY]={datefield:Dygraph.DATEFIELD_SS,step:30,spacing:3e4},Dygraph.TICK_PLACEMENT[Dygraph.MINUTELY]={datefield:Dygraph.DATEFIELD_MM,step:1,spacing:6e4},Dygraph.TICK_PLACEMENT[Dygraph.TWO_MINUTELY]={datefield:Dygraph.DATEFIELD_MM,step:2,spacing:12e4},Dygraph.TICK_PLACEMENT[Dygraph.FIVE_MINUTELY]={datefield:Dygraph.DATEFIELD_MM,step:5,spacing:3e5},Dygraph.TICK_PLACEMENT[Dygraph.TEN_MINUTELY]={datefield:Dygraph.DATEFIELD_MM,step:10,spacing:6e5},Dygraph.TICK_PLACEMENT[Dygraph.THIRTY_MINUTELY]={datefield:Dygraph.DATEFIELD_MM,step:30,spacing:18e5},Dygraph.TICK_PLACEMENT[Dygraph.HOURLY]={datefield:Dygraph.DATEFIELD_HH,step:1,spacing:36e5},Dygraph.TICK_PLACEMENT[Dygraph.TWO_HOURLY]={datefield:Dygraph.DATEFIELD_HH,step:2,spacing:72e5},Dygraph.TICK_PLACEMENT[Dygraph.SIX_HOURLY]={datefield:Dygraph.DATEFIELD_HH,step:6,spacing:216e5},Dygraph.TICK_PLACEMENT[Dygraph.DAILY]={datefield:Dygraph.DATEFIELD_D,step:1,spacing:864e5},Dygraph.TICK_PLACEMENT[Dygraph.TWO_DAILY]={datefield:Dygraph.DATEFIELD_D,step:2,spacing:1728e5},Dygraph.TICK_PLACEMENT[Dygraph.WEEKLY]={datefield:Dygraph.DATEFIELD_D,step:7,spacing:6048e5},Dygraph.TICK_PLACEMENT[Dygraph.MONTHLY]={datefield:Dygraph.DATEFIELD_M,step:1,spacing:2629817280},Dygraph.TICK_PLACEMENT[Dygraph.QUARTERLY]={datefield:Dygraph.DATEFIELD_M,step:3,spacing:216e5*365.2524},Dygraph.TICK_PLACEMENT[Dygraph.BIANNUAL]={datefield:Dygraph.DATEFIELD_M,step:6,spacing:432e5*365.2524},Dygraph.TICK_PLACEMENT[Dygraph.ANNUAL]={datefield:Dygraph.DATEFIELD_Y,step:1,spacing:864e5*365.2524},Dygraph.TICK_PLACEMENT[Dygraph.DECADAL]={datefield:Dygraph.DATEFIELD_Y,step:10,spacing:315578073600},Dygraph.TICK_PLACEMENT[Dygraph.CENTENNIAL]={datefield:Dygraph.DATEFIELD_Y,step:100,spacing:3155780736e3},Dygraph.PREFERRED_LOG_TICK_VALUES=function(){for(var t=[],e=-39;39>=e;e++)for(var a=Math.pow(10,e),i=1;9>=i;i++){var r=a*i;t.push(r)}return t}(),Dygraph.pickDateTickGranularity=function(t,e,a,i){for(var r=i("pixelsPerLabel"),n=0;n=r)return n}return-1},Dygraph.numDateTicks=function(t,e,a){var i=Dygraph.TICK_PLACEMENT[a].spacing;return Math.round(1*(e-t)/i)},Dygraph.getDateAxis=function(t,e,a,i,r){var n=i("axisLabelFormatter"),o=i("labelsUTC"),s=o?Dygraph.DateAccessorsUTC:Dygraph.DateAccessorsLocal,l=Dygraph.TICK_PLACEMENT[a].datefield,h=Dygraph.TICK_PLACEMENT[a].step,p=Dygraph.TICK_PLACEMENT[a].spacing,g=new Date(t),d=[];d[Dygraph.DATEFIELD_Y]=s.getFullYear(g),d[Dygraph.DATEFIELD_M]=s.getMonth(g),d[Dygraph.DATEFIELD_D]=s.getDate(g),d[Dygraph.DATEFIELD_HH]=s.getHours(g),d[Dygraph.DATEFIELD_MM]=s.getMinutes(g),d[Dygraph.DATEFIELD_SS]=s.getSeconds(g),d[Dygraph.DATEFIELD_MS]=s.getMilliseconds(g);var u=d[l]%h;a==Dygraph.WEEKLY&&(u=s.getDay(g)),d[l]-=u;for(var c=l+1;cv&&(v+=p,_=new Date(v));e>=v;)y.push({v:v,label:n.call(r,_,a,i,r)}),v+=p,_=new Date(v);else for(t>v&&(d[l]+=h,_=s.makeDate.apply(null,d),v=_.getTime());e>=v;)(a>=Dygraph.DAILY||s.getHours(_)%h===0)&&y.push({v:v,label:n.call(r,_,a,i,r)}),d[l]+=h,_=s.makeDate.apply(null,d),v=_.getTime();return y},Dygraph&&Dygraph.DEFAULT_ATTRS&&Dygraph.DEFAULT_ATTRS.axes&&Dygraph.DEFAULT_ATTRS.axes.x&&Dygraph.DEFAULT_ATTRS.axes.y&&Dygraph.DEFAULT_ATTRS.axes.y2&&(Dygraph.DEFAULT_ATTRS.axes.x.ticker=Dygraph.dateTicker,Dygraph.DEFAULT_ATTRS.axes.y.ticker=Dygraph.numericTicks,Dygraph.DEFAULT_ATTRS.axes.y2.ticker=Dygraph.numericTicks)}(),Dygraph.Plugins={},Dygraph.Plugins.Annotations=function(){"use strict";var t=function(){this.annotations_=[]};return t.prototype.toString=function(){return"Annotations Plugin"},t.prototype.activate=function(t){return{clearChart:this.clearChart,didDrawChart:this.didDrawChart}},t.prototype.detachLabels=function(){for(var t=0;to.x+o.w||h.canvasyo.y+o.h)){var p=h.annotation,g=6;p.hasOwnProperty("tickHeight")&&(g=p.tickHeight);var d=document.createElement("div");for(var u in r)r.hasOwnProperty(u)&&(d.style[u]=r[u]);p.hasOwnProperty("icon")||(d.className="dygraphDefaultAnnotation"),p.hasOwnProperty("cssClass")&&(d.className+=" "+p.cssClass);var c=p.hasOwnProperty("width")?p.width:16,y=p.hasOwnProperty("height")?p.height:16;if(p.hasOwnProperty("icon")){var _=document.createElement("img");_.src=p.icon,_.width=c,_.height=y,d.appendChild(_)}else h.annotation.hasOwnProperty("shortText")&&d.appendChild(document.createTextNode(h.annotation.shortText));var v=h.canvasx-c/2;d.style.left=v+"px";var f=0;if(p.attachAtBottom){var x=o.y+o.h-y-g;s[v]?x-=s[v]:s[v]=0,s[v]+=g+y,f=x}else f=h.canvasy-y-g;d.style.top=f+"px",d.style.width=c+"px",d.style.height=y+"px",d.title=h.annotation.text,d.style.color=e.colorsMap_[h.name],d.style.borderColor=e.colorsMap_[h.name],p.div=d,e.addAndTrackEvent(d,"click",n("clickHandler","annotationClickHandler",h,this)),e.addAndTrackEvent(d,"mouseover",n("mouseOverHandler","annotationMouseOverHandler",h,this)),e.addAndTrackEvent(d,"mouseout",n("mouseOutHandler","annotationMouseOutHandler",h,this)),e.addAndTrackEvent(d,"dblclick",n("dblClickHandler","annotationDblClickHandler",h,this)),i.appendChild(d),this.annotations_.push(d);var m=t.drawingContext;if(m.save(),m.strokeStyle=e.colorsMap_[h.name],m.beginPath(),p.attachAtBottom){var x=f+y;m.moveTo(h.canvasx,x),m.lineTo(h.canvasx,x+g)}else m.moveTo(h.canvasx,h.canvasy),m.lineTo(h.canvasx,h.canvasy-2-g);m.closePath(),m.stroke(),m.restore()}}},t.prototype.destroy=function(){this.detachLabels()},t}(),Dygraph.Plugins.Axes=function(){"use strict";var t=function(){this.xlabels_=[],this.ylabels_=[]};return t.prototype.toString=function(){return"Axes Plugin"},t.prototype.activate=function(t){return{layout:this.layout,clearChart:this.clearChart,willDrawChart:this.willDrawChart}},t.prototype.layout=function(t){var e=t.dygraph;if(e.getOptionForAxis("drawAxis","y")){var a=e.getOptionForAxis("axisLabelWidth","y")+2*e.getOptionForAxis("axisTickSize","y");t.reserveSpaceLeft(a)}if(e.getOptionForAxis("drawAxis","x")){var i;i=e.getOption("xAxisHeight")?e.getOption("xAxisHeight"):e.getOptionForAxis("axisLabelFontSize","x")+2*e.getOptionForAxis("axisTickSize","x"),t.reserveSpaceBottom(i)}if(2==e.numAxes()){if(e.getOptionForAxis("drawAxis","y2")){var a=e.getOptionForAxis("axisLabelWidth","y2")+2*e.getOptionForAxis("axisTickSize","y2");t.reserveSpaceRight(a)}}else e.numAxes()>2&&e.error("Only two y-axes are supported at this time. (Trying to use "+e.numAxes()+")")},t.prototype.detachLabels=function(){function t(t){for(var e=0;e0){var x=i.numAxes(),m=[f("y"),f("y2")];for(l=0;l<_.yticks.length;l++){if(s=_.yticks[l],"function"==typeof s)return;n=v.x;var D=1,w="y1",A=m[0];1==s[0]&&(n=v.x+v.w,D=-1,w="y2",A=m[1]);var b=A("axisLabelFontSize");o=v.y+s[1]*v.h,r=y(s[2],"y",2==x?w:null);var T=o-b/2;0>T&&(T=0),T+b+3>d?r.style.bottom="0":r.style.top=T+"px",0===s[0]?(r.style.left=v.x-A("axisLabelWidth")-A("axisTickSize")+"px",r.style.textAlign="right"):1==s[0]&&(r.style.left=v.x+v.w+A("axisTickSize")+"px",r.style.textAlign="left"),r.style.width=A("axisLabelWidth")+"px",p.appendChild(r),this.ylabels_.push(r)}var E=this.ylabels_[0],b=i.getOptionForAxis("axisLabelFontSize","y"),C=parseInt(E.style.top,10)+b;C>d-b&&(E.style.top=parseInt(E.style.top,10)-b/2+"px")}var L;if(i.getOption("drawAxesAtZero")){var P=i.toPercentXCoord(0);(P>1||0>P||isNaN(P))&&(P=0),L=e(v.x+P*v.w)}else L=e(v.x);h.strokeStyle=i.getOptionForAxis("axisLineColor","y"),h.lineWidth=i.getOptionForAxis("axisLineWidth","y"),h.beginPath(),h.moveTo(L,a(v.y)),h.lineTo(L,a(v.y+v.h)),h.closePath(),h.stroke(),2==i.numAxes()&&(h.strokeStyle=i.getOptionForAxis("axisLineColor","y2"),h.lineWidth=i.getOptionForAxis("axisLineWidth","y2"),h.beginPath(),h.moveTo(a(v.x+v.w),a(v.y)),h.lineTo(a(v.x+v.w),a(v.y+v.h)),h.closePath(),h.stroke())}if(i.getOptionForAxis("drawAxis","x")){if(_.xticks){var A=f("x");for(l=0;l<_.xticks.length;l++){s=_.xticks[l],n=v.x+s[0]*v.w,o=v.y+v.h,r=y(s[1],"x"),r.style.textAlign="center",r.style.top=o+A("axisTickSize")+"px";var S=n-A("axisLabelWidth")/2;S+A("axisLabelWidth")>g&&(S=g-A("axisLabelWidth"),r.style.textAlign="right"),0>S&&(S=0,r.style.textAlign="left"),r.style.left=S+"px",r.style.width=A("axisLabelWidth")+"px", +p.appendChild(r),this.xlabels_.push(r)}}h.strokeStyle=i.getOptionForAxis("axisLineColor","x"),h.lineWidth=i.getOptionForAxis("axisLineWidth","x"),h.beginPath();var O;if(i.getOption("drawAxesAtZero")){var P=i.toPercentYCoord(0,0);(P>1||0>P)&&(P=1),O=a(v.y+P*v.h)}else O=a(v.y+v.h);h.moveTo(e(v.x),O),h.lineTo(e(v.x+v.w),O),h.closePath(),h.stroke()}h.restore()}},t}(),Dygraph.Plugins.ChartLabels=function(){"use strict";var t=function(){this.title_div_=null,this.xlabel_div_=null,this.ylabel_div_=null,this.y2label_div_=null};t.prototype.toString=function(){return"ChartLabels Plugin"},t.prototype.activate=function(t){return{layout:this.layout,didDrawChart:this.didDrawChart}};var e=function(t){var e=document.createElement("div");return e.style.position="absolute",e.style.left=t.x+"px",e.style.top=t.y+"px",e.style.width=t.w+"px",e.style.height=t.h+"px",e};t.prototype.detachLabels_=function(){for(var t=[this.title_div_,this.xlabel_div_,this.ylabel_div_,this.y2label_div_],e=0;e=2);for(o=h.yticks,l.save(),n=0;n=2;for(y&&l.installPattern(_),l.strokeStyle=s.getOptionForAxis("gridLineColor","x"),l.lineWidth=s.getOptionForAxis("gridLineWidth","x"),n=0;n/g,">")};return t.prototype.select=function(e){var a=e.selectedX,i=e.selectedPoints,r=e.selectedRow,n=e.dygraph.getOption("legend");if("never"===n)return void(this.legend_div_.style.display="none");if("follow"===n){var o=e.dygraph.plotter_.area,s=e.dygraph.getOption("labelsDivWidth"),l=e.dygraph.getOptionForAxis("axisLabelWidth","y"),h=i[0].x*o.w+20,p=i[0].y*o.h-20;h+s+1>window.scrollX+window.innerWidth&&(h=h-40-s-(l-o.x)),e.dygraph.graphDiv.appendChild(this.legend_div_),this.legend_div_.style.left=l+h+"px",this.legend_div_.style.top=p+"px"}var g=t.generateLegendHTML(e.dygraph,a,i,this.one_em_width_,r);this.legend_div_.innerHTML=g,this.legend_div_.style.display=""},t.prototype.deselect=function(e){var i=e.dygraph.getOption("legend");"always"!==i&&(this.legend_div_.style.display="none");var r=a(this.legend_div_);this.one_em_width_=r;var n=t.generateLegendHTML(e.dygraph,void 0,void 0,r,null);this.legend_div_.innerHTML=n},t.prototype.didDrawChart=function(t){this.deselect(t)},t.prototype.predraw=function(t){if(this.is_generated_div_){t.dygraph.graphDiv.appendChild(this.legend_div_);var e=t.dygraph.plotter_.area,a=t.dygraph.getOption("labelsDivWidth");this.legend_div_.style.left=e.x+e.w-a-1+"px",this.legend_div_.style.top=e.y+"px",this.legend_div_.style.width=a+"px"}},t.prototype.destroy=function(){this.legend_div_=null},t.generateLegendHTML=function(t,a,r,n,o){if(t.getOption("showLabelsOnHighlight")!==!0)return"";var s,l,h,p,g,d=t.getLabels();if("undefined"==typeof a){if("always"!=t.getOption("legend"))return"";for(l=t.getOption("labelsSeparateLines"),s="",h=1;h":" "),g=t.getOption("strokePattern",d[h]),p=e(g,u.color,n),s+=""+p+" "+i(d[h])+"")}return s}var c=t.optionsViewForAxis_("x"),y=c("valueFormatter");s=y.call(t,a,c,d[0],t,o,0),""!==s&&(s+=":");var _=[],v=t.numAxes();for(h=0;v>h;h++)_[h]=t.optionsViewForAxis_("y"+(h?1+h:""));var f=t.getOption("labelsShowZeroValues");l=t.getOption("labelsSeparateLines");var x=t.getHighlightSeries();for(h=0;h");var u=t.getPropertiesForSeries(m.name),D=_[u.axis-1],w=D("valueFormatter"),A=w.call(t,m.yval,D,m.name,t,o,d.indexOf(m.name)),b=m.name==x?" class='highlight'":"";s+=" "+i(m.name)+": "+A+""}}return s},e=function(t,e,a){var i=/MSIE/.test(navigator.userAgent)&&!window.opera;if(i)return"—";if(!t||t.length<=1)return'
';var r,n,o,s,l,h=0,p=0,g=[];for(r=0;r<=t.length;r++)h+=t[r%t.length];if(l=Math.floor(a/(h-t[0])),l>1){for(r=0;rn;n++)for(r=0;p>r;r+=2)o=g[r%g.length],s=r';return d},t}(),Dygraph.Plugins.RangeSelector=function(){"use strict";var t=function(){this.isIE_=/MSIE/.test(navigator.userAgent)&&!window.opera,this.hasTouchInterface_="undefined"!=typeof TouchEvent,this.isMobileDevice_=/mobile|android/gi.test(navigator.appVersion),this.interfaceCreated_=!1};return t.prototype.toString=function(){return"RangeSelector Plugin"},t.prototype.activate=function(t){return this.dygraph_=t,this.isUsingExcanvas_=t.isUsingExcanvas_,this.getOption_("showRangeSelector")&&this.createInterface_(),{layout:this.reserveSpace_,predraw:this.renderStaticLayer_,didDrawChart:this.renderInteractiveLayer_}},t.prototype.destroy=function(){this.bgcanvas_=null,this.fgcanvas_=null,this.leftZoomHandle_=null,this.rightZoomHandle_=null,this.iePanOverlay_=null},t.prototype.getOption_=function(t,e){return this.dygraph_.getOption(t,e)},t.prototype.setDefaultOption_=function(t,e){this.dygraph_.attrs_[t]=e},t.prototype.createInterface_=function(){this.createCanvases_(),this.isUsingExcanvas_&&this.createIEPanOverlay_(),this.createZoomHandles_(),this.initInteraction_(),this.getOption_("animatedZooms")&&(console.warn("Animated zooms and range selector are not compatible; disabling animatedZooms."),this.dygraph_.updateOptions({animatedZooms:!1},!0)),this.interfaceCreated_=!0,this.addToGraph_()},t.prototype.addToGraph_=function(){var t=this.graphDiv_=this.dygraph_.graphDiv;t.appendChild(this.bgcanvas_),t.appendChild(this.fgcanvas_),t.appendChild(this.leftZoomHandle_),t.appendChild(this.rightZoomHandle_)},t.prototype.removeFromGraph_=function(){var t=this.graphDiv_;t.removeChild(this.bgcanvas_),t.removeChild(this.fgcanvas_),t.removeChild(this.leftZoomHandle_),t.removeChild(this.rightZoomHandle_),this.graphDiv_=null},t.prototype.reserveSpace_=function(t){this.getOption_("showRangeSelector")&&t.reserveSpaceBottom(this.getOption_("rangeSelectorHeight")+4)},t.prototype.renderStaticLayer_=function(){this.updateVisibility_()&&(this.resize_(),this.drawStaticLayer_())},t.prototype.renderInteractiveLayer_=function(){this.updateVisibility_()&&!this.isChangingRange_&&(this.placeZoomHandles_(),this.drawInteractiveLayer_())},t.prototype.updateVisibility_=function(){var t=this.getOption_("showRangeSelector");if(t)this.interfaceCreated_?this.graphDiv_&&this.graphDiv_.parentNode||this.addToGraph_():this.createInterface_();else if(this.graphDiv_){this.removeFromGraph_();var e=this.dygraph_;setTimeout(function(){e.width_=0,e.resize()},1)}return t},t.prototype.resize_=function(){function t(t,e,a){var i=Dygraph.getContextPixelRatio(e);t.style.top=a.y+"px",t.style.left=a.x+"px",t.width=a.w*i,t.height=a.h*i,t.style.width=a.w+"px",t.style.height=a.h+"px",1!=i&&e.scale(i,i)}var e=this.dygraph_.layout_.getPlotArea(),a=0;this.dygraph_.getOptionForAxis("drawAxis","x")&&(a=this.getOption_("xAxisHeight")||this.getOption_("axisLabelFontSize")+2*this.getOption_("axisTickSize")),this.canvasRect_={x:e.x,y:e.y+e.h+a+4,w:e.w,h:this.getOption_("rangeSelectorHeight")},t(this.bgcanvas_,this.bgcanvas_ctx_,this.canvasRect_),t(this.fgcanvas_,this.fgcanvas_ctx_,this.canvasRect_)},t.prototype.createCanvases_=function(){this.bgcanvas_=Dygraph.createCanvas(),this.bgcanvas_.className="dygraph-rangesel-bgcanvas",this.bgcanvas_.style.position="absolute",this.bgcanvas_.style.zIndex=9,this.bgcanvas_ctx_=Dygraph.getContext(this.bgcanvas_),this.fgcanvas_=Dygraph.createCanvas(),this.fgcanvas_.className="dygraph-rangesel-fgcanvas",this.fgcanvas_.style.position="absolute",this.fgcanvas_.style.zIndex=9,this.fgcanvas_.style.cursor="default",this.fgcanvas_ctx_=Dygraph.getContext(this.fgcanvas_)},t.prototype.createIEPanOverlay_=function(){this.iePanOverlay_=document.createElement("div"),this.iePanOverlay_.style.position="absolute",this.iePanOverlay_.style.backgroundColor="white",this.iePanOverlay_.style.filter="alpha(opacity=0)",this.iePanOverlay_.style.display="none",this.iePanOverlay_.style.cursor="move",this.fgcanvas_.appendChild(this.iePanOverlay_)},t.prototype.createZoomHandles_=function(){var t=new Image;t.className="dygraph-rangesel-zoomhandle",t.style.position="absolute",t.style.zIndex=10,t.style.visibility="hidden",t.style.cursor="col-resize",/MSIE 7/.test(navigator.userAgent)?(t.width=7,t.height=14,t.style.backgroundColor="white",t.style.border="1px solid #333333"):(t.width=9,t.height=16,t.src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAkAAAAQCAYAAADESFVDAAAAAXNSR0IArs4c6QAAAAZiS0dEANAAzwDP4Z7KegAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAAd0SU1FB9sHGw0cMqdt1UwAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAaElEQVQoz+3SsRFAQBCF4Z9WJM8KCDVwownl6YXsTmCUsyKGkZzcl7zkz3YLkypgAnreFmDEpHkIwVOMfpdi9CEEN2nGpFdwD03yEqDtOgCaun7sqSTDH32I1pQA2Pb9sZecAxc5r3IAb21d6878xsAAAAAASUVORK5CYII="),this.isMobileDevice_&&(t.width*=2,t.height*=2),this.leftZoomHandle_=t,this.rightZoomHandle_=t.cloneNode(!1)},t.prototype.initInteraction_=function(){var t,e,a,i,r,n,o,s,l,h,p,g,d,u,c=this,y=document,_=0,v=null,f=!1,x=!1,m=!this.isMobileDevice_&&!this.isUsingExcanvas_,D=new Dygraph.IFrameTarp;t=function(t){var e=c.dygraph_.xAxisExtremes(),a=(e[1]-e[0])/c.canvasRect_.w,i=e[0]+(t.leftHandlePos-c.canvasRect_.x)*a,r=e[0]+(t.rightHandlePos-c.canvasRect_.x)*a;return[i,r]},e=function(t){return Dygraph.cancelEvent(t),f=!0,_=t.clientX,v=t.target?t.target:t.srcElement,("mousedown"===t.type||"dragstart"===t.type)&&(Dygraph.addEvent(y,"mousemove",a),Dygraph.addEvent(y,"mouseup",i)),c.fgcanvas_.style.cursor="col-resize",D.cover(),!0},a=function(t){if(!f)return!1;Dygraph.cancelEvent(t);var e=t.clientX-_;if(Math.abs(e)<4)return!0;_=t.clientX;var a,i=c.getZoomHandleStatus_();v==c.leftZoomHandle_?(a=i.leftHandlePos+e,a=Math.min(a,i.rightHandlePos-v.width-3),a=Math.max(a,c.canvasRect_.x)):(a=i.rightHandlePos+e,a=Math.min(a,c.canvasRect_.x+c.canvasRect_.w),a=Math.max(a,i.leftHandlePos+v.width+3));var n=v.width/2;return v.style.left=a-n+"px",c.drawInteractiveLayer_(),m&&r(),!0},i=function(t){return f?(f=!1,D.uncover(),Dygraph.removeEvent(y,"mousemove",a),Dygraph.removeEvent(y,"mouseup",i),c.fgcanvas_.style.cursor="default",m||r(),!0):!1},r=function(){try{var e=c.getZoomHandleStatus_();if(c.isChangingRange_=!0,e.isZoomed){var a=t(e);c.dygraph_.doZoomXDates_(a[0],a[1])}else c.dygraph_.resetZoom()}finally{c.isChangingRange_=!1}},n=function(t){if(c.isUsingExcanvas_)return t.srcElement==c.iePanOverlay_;var e=c.leftZoomHandle_.getBoundingClientRect(),a=e.left+e.width/2;e=c.rightZoomHandle_.getBoundingClientRect();var i=e.left+e.width/2;return t.clientX>a&&t.clientX=c.canvasRect_.x+c.canvasRect_.w?(r=c.canvasRect_.x+c.canvasRect_.w,i=r-n):(i+=e,r+=e);var o=c.leftZoomHandle_.width/2;return c.leftZoomHandle_.style.left=i-o+"px",c.rightZoomHandle_.style.left=r-o+"px",c.drawInteractiveLayer_(),m&&h(),!0},l=function(t){return x?(x=!1,Dygraph.removeEvent(y,"mousemove",s),Dygraph.removeEvent(y,"mouseup",l),m||h(),!0):!1},h=function(){try{c.isChangingRange_=!0,c.dygraph_.dateWindow_=t(c.getZoomHandleStatus_()),c.dygraph_.drawGraph_(!1)}finally{c.isChangingRange_=!1}},p=function(t){if(!f&&!x){var e=n(t)?"move":"default";e!=c.fgcanvas_.style.cursor&&(c.fgcanvas_.style.cursor=e)}},g=function(t){"touchstart"==t.type&&1==t.targetTouches.length?e(t.targetTouches[0])&&Dygraph.cancelEvent(t):"touchmove"==t.type&&1==t.targetTouches.length?a(t.targetTouches[0])&&Dygraph.cancelEvent(t):i(t)},d=function(t){"touchstart"==t.type&&1==t.targetTouches.length?o(t.targetTouches[0])&&Dygraph.cancelEvent(t):"touchmove"==t.type&&1==t.targetTouches.length?s(t.targetTouches[0])&&Dygraph.cancelEvent(t):l(t)},u=function(t,e){for(var a=["touchstart","touchend","touchmove","touchcancel"],i=0;it;t++){var s=this.getOption_("showInRangeSelector",r[t]);n[t]=s,null!==s&&(o=!0)}if(!o)for(t=0;t1&&(g=h.rollingAverage(g,e.rollPeriod(),p)),l.push(g)}var d=[];for(t=0;t0)&&(v=Math.min(v,x),f=Math.max(f,x))}var m=.25;if(a)for(f=Dygraph.log10(f),f+=f*m,v=Dygraph.log10(v),t=0;tthis.canvasRect_.x||a+10&&t[r][0]>o;)i--,r--}return i>=a?[a,i]:[0,t.length-1]},t.parseFloat=function(t){return null===t?0/0:t}}(),function(){"use strict";Dygraph.DataHandlers.DefaultHandler=function(){};var t=Dygraph.DataHandlers.DefaultHandler;t.prototype=new Dygraph.DataHandler,t.prototype.extractSeries=function(t,e,a){for(var i=[],r=a.get("logscale"),n=0;n=s&&(s=null),i.push([o,s])}return i},t.prototype.rollingAverage=function(t,e,a){e=Math.min(e,t.length);var i,r,n,o,s,l=[];if(1==e)return t;for(i=0;ir;r++)n=t[r][1],null===n||isNaN(n)||(s++,o+=t[r][1]);s?l[i]=[t[i][0],o/s]:l[i]=[t[i][0],null]}return l},t.prototype.getExtremeYValues=function(t,e,a){for(var i,r=null,n=null,o=0,s=t.length-1,l=o;s>=l;l++)i=t[l][1],null===i||isNaN(i)||((null===n||i>n)&&(n=i),(null===r||r>i)&&(r=i));return[r,n]}}(),function(){"use strict";Dygraph.DataHandlers.DefaultFractionHandler=function(){};var t=Dygraph.DataHandlers.DefaultFractionHandler;t.prototype=new Dygraph.DataHandlers.DefaultHandler,t.prototype.extractSeries=function(t,e,a){for(var i,r,n,o,s,l,h=[],p=100,g=a.get("logscale"),d=0;d=0&&(n-=t[i-e][2][0],o-=t[i-e][2][1]);var l=t[i][0],h=o?n/o:0;r[i]=[l,s*h]}return r}}(),function(){"use strict";Dygraph.DataHandlers.BarsHandler=function(){Dygraph.DataHandler.call(this)},Dygraph.DataHandlers.BarsHandler.prototype=new Dygraph.DataHandler;var t=Dygraph.DataHandlers.BarsHandler;t.prototype.extractSeries=function(t,e,a){},t.prototype.rollingAverage=function(t,e,a){},t.prototype.onPointsCreated_=function(t,e){for(var a=0;a=l;l++)if(i=t[l][1],null!==i&&!isNaN(i)){var h=t[l][2][0],p=t[l][2][1];h>i&&(h=i),i>p&&(p=i),(null===n||p>n)&&(n=p),(null===r||r>h)&&(r=h)}return[r,n]},t.prototype.onLineEvaluated=function(t,e,a){for(var i,r=0;r=0){var g=t[l-e];null===g[1]||isNaN(g[1])||(r-=g[2][0],o-=g[1],n-=g[2][1],s-=1)}s?p[l]=[t[l][0],1*o/s,[1*r/s,1*n/s]]:p[l]=[t[l][0],null,[null,null]]}return p}}(),function(){"use strict";Dygraph.DataHandlers.ErrorBarsHandler=function(){};var t=Dygraph.DataHandlers.ErrorBarsHandler;t.prototype=new Dygraph.DataHandlers.BarsHandler,t.prototype.extractSeries=function(t,e,a){for(var i,r,n,o,s=[],l=a.get("sigma"),h=a.get("logscale"),p=0;pr;r++)n=t[r][1],null===n||isNaN(n)||(l++,s+=n,p+=Math.pow(t[r][2][2],2));l?(h=Math.sqrt(p)/l,g=s/l,d[i]=[t[i][0],g,[g-u*h,g+u*h]]):(o=1==e?t[i][1]:null,d[i]=[t[i][0],o,[o,o]])}return d}}(),function(){"use strict";Dygraph.DataHandlers.FractionsBarsHandler=function(){};var t=Dygraph.DataHandlers.FractionsBarsHandler;t.prototype=new Dygraph.DataHandlers.BarsHandler,t.prototype.extractSeries=function(t,e,a){for(var i,r,n,o,s,l,h,p,g=[],d=100,u=a.get("sigma"),c=a.get("logscale"),y=0;y=0&&(p-=t[n-e][2][2],g-=t[n-e][2][3]);var u=t[n][0],c=g?p/g:0;if(h)if(g){var y=0>c?0:c,_=g,v=l*Math.sqrt(y*(1-y)/_+l*l/(4*_*_)),f=1+l*l/g;i=(y+l*l/(2*g)-v)/f,r=(y+l*l/(2*g)+v)/f,s[n]=[u,y*d,[i*d,r*d]]}else s[n]=[u,0,[0,0]];else o=g?l*Math.sqrt(c*(1-c)/g):1,s[n]=[u,d*c,[d*(c-o),d*(c+o)]]}return s}}(); +//# sourceMappingURL=dygraph-combined.js.map \ No newline at end of file diff --git a/ui/README.md b/ui/README.md new file mode 100644 index 00000000..1a00c1d6 --- /dev/null +++ b/ui/README.md @@ -0,0 +1,6 @@ +ui +== + +This directory contains static HTML, CSS, and JavaScript for the RAPPOR +dashboard. See the `pipeline/` directory for more details. + diff --git a/ui/assoc-day.html b/ui/assoc-day.html new file mode 100644 index 00000000..2255325f --- /dev/null +++ b/ui/assoc-day.html @@ -0,0 +1,44 @@ + + + + Single Day Association Results + + + + + + + + + +

+ + +

+ Home / + Association Overview +

+ + + +

+ + +
+ +

+ + Underlying data: assoc-results.csv +

+ + + + + + diff --git a/ui/assoc-metric.html b/ui/assoc-metric.html new file mode 100644 index 00000000..1ac1dded --- /dev/null +++ b/ui/assoc-metric.html @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + +

+ +

+ Home / + Association Overview +

+ +

+ +

+ + +
+ +

+ + Underlying data: +

+ + + + + + diff --git a/ui/assoc-overview.html b/ui/assoc-overview.html new file mode 100644 index 00000000..e3f06e16 --- /dev/null +++ b/ui/assoc-overview.html @@ -0,0 +1,43 @@ + + + + RAPPOR Association Analysis Overview + + + + + + + + + +

+ +

+ Single variable analysis (latest) +

+ +

+ Home / + Association Overview +

+ +

RAPPOR Association Analysis Overview

+ + +
+ +

+ Underlying data: overview.csv +

+ + + + + + diff --git a/ui/assoc-pair.html b/ui/assoc-pair.html new file mode 100644 index 00000000..7625966a --- /dev/null +++ b/ui/assoc-pair.html @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + +

+ +

+ Home / + Association Overview +

+ +

+ +

+ +

Task Status

+ + +
+ +

+ + Underlying data: +

+ + + + + + diff --git a/ui/day.html b/ui/day.html new file mode 100644 index 00000000..624778c6 --- /dev/null +++ b/ui/day.html @@ -0,0 +1,49 @@ + + + + Single Day Results + + + + + + + + + +

+ + +

+ Home / + Overview / + Histograms +

+ + + +

+ + +
+ +

+ Residuals +

+ +

+ + Underlying data: results.csv +

+ + + + + + diff --git a/ui/histograms.html b/ui/histograms.html new file mode 100644 index 00000000..cce5ee26 --- /dev/null +++ b/ui/histograms.html @@ -0,0 +1,48 @@ + + + + RAPPOR Task Histograms + + + + + + + +

+ Home / + Overview / + Histograms +

+ +

RAPPOR Task Histograms

+ +

Each task's input is a (metric, day), i.e. it runs on the summed reports + for a single metric received in a single day.

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ + + diff --git a/ui/home.html b/ui/home.html new file mode 100644 index 00000000..d4f947aa --- /dev/null +++ b/ui/home.html @@ -0,0 +1,16 @@ + + + + Rappor HOME + + + + + + +

+ Redirecting to https://github.com/google/rappor +

+ + diff --git a/ui/metric.html b/ui/metric.html new file mode 100644 index 00000000..ac14a889 --- /dev/null +++ b/ui/metric.html @@ -0,0 +1,83 @@ + + + + Metric Results + + + + + + + + + + + +

+ +

+ Home / + Overview / + Histograms +

+ + + +

+ +

+ +

Estimated Proportions

+

NOTE: Only the top 5 values for each day are shown

+ + +

+

+ Underlying data: dist.csv +

+ +

Number of Reports

+ +

+ + +

Unallocated Mass

+ +

+ +

+ Plot Help: Drag horizontally to zoom to selection. Double click + to zoom out. Shift + drag to pan. +

+ +

Task Status

+ + +
+ +

+ + Underlying data: status.csv +

+ + + + + + diff --git a/ui/overview.html b/ui/overview.html new file mode 100644 index 00000000..464f983d --- /dev/null +++ b/ui/overview.html @@ -0,0 +1,59 @@ + + + + RAPPOR Results Overview + + + + + + + + + +

+ +

+ Association analysis (latest) +

+ +

+ Home / + Overview / + Histograms +

+ +

RAPPOR Results Overview

+ + +
+ +

+ Underlying data: overview.csv +

+ +

Metric Descriptions

+ + + + + + + + + + + + +
Metric NameOwnersDescription
+ + + + + + diff --git a/ui/table-lib.js b/ui/table-lib.js new file mode 100644 index 00000000..64913dc2 --- /dev/null +++ b/ui/table-lib.js @@ -0,0 +1,482 @@ +// Sortable HTML table. +// +// Usage: +// +// Each page should have gTableStates and gUrlHash variables. This library +// only provides functions / classes, not instances. +// +// Then use these public functions on those variables. They should be hooked +// up to initialization and onhashchange events. +// +// - makeTablesSortable +// - updateTables +// +// Life of a click +// +// - query existing TableState object to find the new state +// - mutate urlHash +// - location.hash = urlHash.encode() +// - onhashchange +// - decode location.hash into urlHash +// - update DOM +// +// HTML generation requirements: +// - +// - need for types. +// - For numbers, class="num-cell" as well as +// - single and + +'use strict'; + +function appendMessage(elem, msg) { + // TODO: escape HTML? + elem.innerHTML += msg + '
'; +} + +function userError(errElem, msg) { + if (errElem) { + appendMessage(errElem, msg); + } else { + console.log(msg); + } +} + +// +// Key functions for column ordering +// +// TODO: better naming convention? + +function identity(x) { + return x; +} + +function lowerCase(x) { + return x.toLowerCase(); +} + +// Parse as number. +function asNumber(x) { + var stripped = x.replace(/[ \t\r\n]/g, ''); + if (stripped === 'NA') { + // return lowest value, so NA sorts below everything else. + return -Number.MAX_VALUE; + } + var numClean = x.replace(/[$,]/g, ''); // remove dollar signs and commas + return parseFloat(numClean); +} + +// as a date. +// +// TODO: Parse into JS date object? +// http://stackoverflow.com/questions/19430561/how-to-sort-a-javascript-array-of-objects-by-date +// Uses getTime(). Hm. + +function asDate(x) { + return x; +} + +// +// Table Implementation +// + +// Given a column array and a key function, construct a permutation of the +// indices [0, n). +function makePermutation(colArray, keyFunc) { + var pairs = []; // (index, result of keyFunc on cell) + + var n = colArray.length; + for (var i = 0; i < n; ++i) { + var value = colArray[i]; + + // NOTE: This could be a URL, so you need to extract that? + // If it's a URL, take the anchor text I guess. + var key = keyFunc(value); + + pairs.push([key, i]); + } + + // Sort by computed key + pairs.sort(function(a, b) { + if (a[0] < b[0]) { + return -1; + } else if (a[0] > b[0]) { + return 1; + } else { + return 0; + } + }); + + // Extract the permutation as second column + var perm = []; + for (var i = 0; i < pairs.length; ++i) { + perm.push(pairs[i][1]); // append index + } + return perm; +} + +function extractCol(rows, colIndex) { + var colArray = []; + for (var i = 0; i < rows.length; ++i) { + var row = rows[i]; + colArray.push(row.cells[colIndex].textContent); + } + return colArray; +} + +// Given an array of DOM row objects, and a list of sort functions (one per +// column), return a list of permutations. +// +// Right now this is eager. Could be lazy later. +function makeAllPermutations(rows, keyFuncs) { + var numCols = keyFuncs.length; + var permutations = []; + for (var i = 0; i < numCols; ++i) { + var colArray = extractCol(rows, i); + var keyFunc = keyFuncs[i]; + var p = makePermutation(colArray, keyFunc); + permutations.push(p); + } + return permutations; +} + +// Model object for a table. (Mostly) independent of the DOM. +function TableState(table, keyFuncs) { + this.table = table; + keyFuncs = keyFuncs || []; // array of column + + // these are mutated + this.sortCol = -1; // not sorted by any col + this.ascending = false; // if sortCol is sorted in ascending order + + if (table === null) { // hack so we can pass dummy table + console.log('TESTING'); + return; + } + + var bodyRows = table.tBodies[0].rows; + this.orig = []; // pointers to row objects in their original order + for (var i = 0; i < bodyRows.length; ++i) { + this.orig.push(bodyRows[i]); + } + + this.colElems = []; + var colgroup = table.getElementsByTagName('colgroup')[0]; + + // copy it into an array + if (!colgroup) { + throw new Error('is required'); + } + + for (var i = 0; i < colgroup.children.length; ++i) { + var colElem = colgroup.children[i]; + var colType = colElem.getAttribute('type'); + var keyFunc; + switch (colType) { + case 'case-sensitive': + keyFunc = identity; + break; + case 'case-insensitive': + keyFunc = lowerCase; + break; + case 'number': + keyFunc = asNumber; + break; + case 'date': + keyFunc = asDate; + break; + default: + throw new Error('Invalid column type ' + colType); + } + keyFuncs[i] = keyFunc; + + this.colElems.push(colElem); + } + + this.permutations = makeAllPermutations(this.orig, keyFuncs); +} + +// Reset sort state. +TableState.prototype.resetSort = function() { + this.sortCol = -1; // not sorted by any col + this.ascending = false; // if sortCol is sorted in ascending order +}; + +// Change state for a click on a column. +TableState.prototype.doClick = function(colIndex) { + if (this.sortCol === colIndex) { // same column; invert direction + this.ascending = !this.ascending; + } else { // different column + this.sortCol = colIndex; + // first click makes it *descending*. Typically you want to see the + // largest values first. + this.ascending = false; + } +}; + +TableState.prototype.decode = function(stateStr, errElem) { + var sortCol = parseInt(stateStr); // parse leading integer + var lastChar = stateStr[stateStr.length - 1]; + + var ascending; + if (lastChar === 'a') { + ascending = true; + } else if (lastChar === 'd') { + ascending = false; + } else { + // The user could have entered a bad ID + userError(errElem, 'Invalid state string ' + stateStr); + return; + } + + this.sortCol = sortCol; + this.ascending = ascending; +} + + +TableState.prototype.encode = function() { + if (this.sortCol === -1) { + return ''; // default state isn't serialized + } + + var s = this.sortCol.toString(); + s += this.ascending ? 'a' : 'd'; + return s; +}; + +// Update the DOM with using this object's internal state. +TableState.prototype.updateDom = function() { + var tHead = this.table.tHead; + setArrows(tHead, this.sortCol, this.ascending); + + // Highlight the column that the table is sorted by. + for (var i = 0; i < this.colElems.length; ++i) { + // set or clear it. NOTE: This means we can't have other classes on the + // tags, which is OK. + var className = (i === this.sortCol) ? 'highlight' : ''; + this.colElems[i].className = className; + } + + var n = this.orig.length; + var tbody = this.table.tBodies[0]; + + if (this.sortCol === -1) { // reset it and return + for (var i = 0; i < n; ++i) { + tbody.appendChild(this.orig[i]); + } + return; + } + + var perm = this.permutations[this.sortCol]; + if (this.ascending) { + for (var i = 0; i < n; ++i) { + var index = perm[i]; + tbody.appendChild(this.orig[index]); + } + } else { // descending, apply the permutation in reverse order + for (var i = n - 1; i >= 0; --i) { + var index = perm[i]; + tbody.appendChild(this.orig[index]); + } + } +}; + +var kTablePrefix = 't:'; +var kTablePrefixLength = 2; + +// Given a UrlHash instance and a list of tables, mutate tableStates. +function decodeState(urlHash, tableStates, errElem) { + var keys = urlHash.getKeysWithPrefix(kTablePrefix); // by convention, t:foo=1a + for (var i = 0; i < keys.length; ++i) { + var key = keys[i]; + var tableId = key.substring(kTablePrefixLength); + + if (!tableStates.hasOwnProperty(tableId)) { + // The user could have entered a bad ID + userError(errElem, 'Invalid table ID [' + tableId + ']'); + return; + } + + var state = tableStates[tableId]; + var stateStr = urlHash.get(key); // e.g. '1d' + + state.decode(stateStr, errElem); + } +} + +// Add element for sort arrows. +function addArrowSpans(tHead) { + var tHeadCells = tHead.rows[0].cells; + for (var i = 0; i < tHeadCells.length; ++i) { + var colHead = tHeadCells[i]; + // Put a space in so the width is relatively constant + colHead.innerHTML += '  '; + } +} + +// Go through all the cells in the header. Clear the arrow if there is one. +// Set the one on the correct column. +// +// How to do this? Each column needs a modify the text? +function setArrows(tHead, sortCol, ascending) { + var tHeadCells = tHead.rows[0].cells; + + for (var i = 0; i < tHeadCells.length; ++i) { + var colHead = tHeadCells[i]; + var span = colHead.getElementsByTagName('span')[0]; + + if (i === sortCol) { + span.innerHTML = ascending ? '▴' : '▾'; + } else { + span.innerHTML = ' '; // clear it + } + } +} + +// Given the URL hash, table states, tableId, and column index that was +// clicked, visit a new location. +function makeClickHandler(urlHash, tableStates, id, colIndex) { + return function() { // no args for onclick= + var clickedState = tableStates[id]; + + clickedState.doClick(colIndex); + + // now urlHash has non-table state, and tableStates is the table state. + for (var tableId in tableStates) { + var state = tableStates[tableId]; + + var stateStr = state.encode(); + var key = kTablePrefix + tableId; + + if (stateStr === '') { + urlHash.del(key); + } else { + urlHash.set(key, stateStr); + } + } + + // move to new location + location.hash = urlHash.encode(); + }; +} + +// Go through cells and register onClick +function registerClick(table, urlHash, tableStates) { + var id = table.id; // id is required + + var tHeadCells = table.tHead.rows[0].cells; + for (var colIndex = 0; colIndex < tHeadCells.length; ++colIndex) { + var colHead = tHeadCells[colIndex]; + // NOTE: in ES5, could use 'bind'. + colHead.onclick = makeClickHandler(urlHash, tableStates, id, colIndex); + } +} + +// +// Public Functions (TODO: Make a module?) +// + +// Parse the URL fragment, and update all tables. Errors are printed to a DOM +// element. +function updateTables(urlHash, tableStates, statusElem) { + // State should come from the hash alone, so reset old state. (We want to + // keep the permutations though.) + for (var tableId in tableStates) { + tableStates[tableId].resetSort(); + } + + decodeState(urlHash, tableStates, statusElem); + + for (var name in tableStates) { + var state = tableStates[name]; + state.updateDom(); + } +} + +// Takes a {tableId: spec} object. The spec should be an array of sortable +// items. +// Returns a dictionary of table states. +function makeTablesSortable(urlHash, tables, tableStates) { + for (var i = 0; i < tables.length; ++i) { + var table = tables[i]; + var tableId = table.id; + + registerClick(table, urlHash, tableStates); + tableStates[tableId] = new TableState(table); + + addArrowSpans(table.tHead); + } + return tableStates; +} + +// table-sort.js can use t:holidays=1d +// +// metric.html can use: +// +// metric=Foo.bar +// +// day.html could use +// +// jobId=X&metric=Foo.bar&day=2015-06-01 + +// helper +function _decode(s) { + var obj = {}; + var parts = s.split('&'); + for (var i = 0; i < parts.length; ++i) { + if (parts[i].length === 0) { + continue; // quirk: ''.split('&') is [''] ? Should be a 0-length array. + } + var pair = parts[i].split('='); + obj[pair[0]] = pair[1]; // for now, assuming no = + } + return obj; +} + +// UrlHash Constructor. +// Args: +// hashStr: location.hash +function UrlHash(hashStr) { + this.reset(hashStr); +} + +UrlHash.prototype.reset = function(hashStr) { + var h = hashStr.substring(1); // without leading # + // Internal storage is string -> string + this.dict = _decode(h); +} + +UrlHash.prototype.set = function(name, value) { + this.dict[name] = value; +}; + +UrlHash.prototype.del = function(name) { + delete this.dict[name]; +}; + +UrlHash.prototype.get = function(name ) { + return this.dict[name]; +}; + +// e.g. Table states have keys which start with 't:'. +UrlHash.prototype.getKeysWithPrefix = function(prefix) { + var keys = []; + for (var name in this.dict) { + if (name.indexOf(prefix) === 0) { + keys.push(name); + } + } + return keys; +}; + +// Return a string reflecting internal key-value pairs. +UrlHash.prototype.encode = function() { + var parts = []; + for (var name in this.dict) { + var s = name; + s += '='; + var value = this.dict[name]; + s += encodeURIComponent(value); + parts.push(s); + } + return parts.join('&'); +}; diff --git a/ui/table-sort.css b/ui/table-sort.css new file mode 100644 index 00000000..1034f4e5 --- /dev/null +++ b/ui/table-sort.css @@ -0,0 +1,39 @@ +/* sort indicator in column headings */ +.sortArrow { + color: grey; +} + +thead { + font-weight: bold; + text-align: center; +} + +table { + padding: 10px; /* Padding makes it look nicer. */ + margin: 0 auto; /* center table on the page */ + border-collapse: collapse; /* this is like old cellpadding */ +} + +/* like cellspacing? */ +td { + padding: 5px; +} + +/* Built-in support for R NA values */ +.na { + color: darkred; +} + +/* Numbers aligned on the right, like Excel */ +.num { + text-align: right; +} + +.highlight { + background-color: #f0f0f0; +} + +tbody tr:hover { + background-color: lightcyan; +} + diff --git a/ui/ui.css b/ui/ui.css new file mode 100644 index 00000000..8431ecf0 --- /dev/null +++ b/ui/ui.css @@ -0,0 +1,53 @@ +/* Center the plots */ +.dy { + margin: 0 auto; + width: 50em; +} + +/* main metric */ +#proportionsDy { + width: 1000px; + height: 600px; +} + +#num-reports-dy { + width: 1000px; + height: 300px; +} + +#mass-dy { + width: 1000px; + height: 300px; +} + +#metricDesc { + font-style: italic; +} + +body { + /*margin: 0 auto;*/ + /*text-align: left;*/ +} + +h1 { + text-align: center; +} + +h2 { + text-align: center; +} + +p { + text-align: center; +} + +/* R NA values */ +.na { + color: darkred; +} + +#status { + text-align: center; + font-size: x-large; + color: darkred; +} diff --git a/ui/ui.js b/ui/ui.js new file mode 100644 index 00000000..b74a8e2e --- /dev/null +++ b/ui/ui.js @@ -0,0 +1,363 @@ +// Dashboard UI functions. +// +// This is shared between all HTML pages. + +'use strict'; + +// Append a message to an element. Used for errors. +function appendMessage(elem, msg) { + elem.innerHTML += msg + '
'; +} + +// jQuery-like AJAX helper, but simpler. + +// Requires an element with id "status" to show errors. +// +// Args: +// errElem: optional element to append error messages to. If null, then +// alert() on error. +// success: callback that is passed the xhr object. +function ajaxGet(url, errElem, success) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true /*async*/); + xhr.onreadystatechange = function() { + if (xhr.readyState != 4 /*DONE*/) { + return; + } + + if (xhr.status != 200) { + var msg = 'ERROR requesting ' + url + ': ' + xhr.status + ' ' + + xhr.statusText; + if (errElem) { + appendMessage(errElem, msg); + } else { + alert(msg); + } + return; + } + + success(xhr); + }; + xhr.send(); +} + +// Load metadata about the metrics. +// metric-metadata.json is just 14 KB, so we load it for every page. +// +// callback: +// on metric page, just pick out the right description. +// on overview page, populate them ALL with tool tips? +// Or create another column? +function loadMetricMetadata(errElem, success) { + // TODO: Should we make metric-metadata.json optional? Some may not have it. + + ajaxGet('metric-metadata.json', errElem, function(xhr) { + // TODO: handle parse error + var m = JSON.parse(xhr.responseText); + success(m); + }); +} + +// for overview.html. +function initOverview(urlHash, tableStates, statusElem) { + + ajaxGet('cooked/overview.part.html', statusElem, function(xhr) { + var elem = document.getElementById('overview'); + elem.innerHTML = xhr.responseText; + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); + + loadMetricMetadata(statusElem, function(metadata) { + var elem = document.getElementById('metricMetadata').tBodies[0]; + var metrics = metadata.metrics; + + // Sort by the metric name + var metricNames = Object.getOwnPropertyNames(metrics); + metricNames.sort(); + + var tableHtml = ''; + for (var i = 0; i < metricNames.length; ++i) { + var name = metricNames[i]; + var meta = metrics[name]; + tableHtml += ''; + tableHtml += ''; + tableHtml += ''; + tableHtml += ''; + tableHtml += ''; + } + elem.innerHTML += tableHtml; + }); +} + +// for metric.html. +function initMetric(urlHash, tableStates, statusElem, globals) { + + var metricName = urlHash.get('metric'); + if (metricName === undefined) { + appendMessage(statusElem, "Missing metric name in URL hash."); + return; + } + + loadMetricMetadata(statusElem, function(metadata) { + var meta = metadata.metrics[metricName]; + if (!meta) { + appendMessage(statusElem, 'Found no metadata for ' + metricName); + return; + } + var descElem = document.getElementById('metricDesc'); + descElem.innerHTML = meta.summary; + + // TODO: put owners at the bottom of the page somewhere? + }); + + // Add title and page element + document.title = metricName; + var nameElem = document.getElementById('metricName'); + nameElem.innerHTML = metricName; + + // Add correct links. + var u = document.getElementById('underlying-status'); + u.href = 'cooked/' + metricName + '/status.csv'; + + var distUrl = 'cooked/' + metricName + '/dist.csv'; + var u2 = document.getElementById('underlying-dist'); + u2.href = distUrl; + + ajaxGet(distUrl, statusElem, function(xhr) { + var csvData = xhr.responseText; + var elem = document.getElementById('proportionsDy'); + // Mutate global so we can respond to onclick. + globals.proportionsDygraph = new Dygraph(elem, csvData, {customBars: true}); + }); + + var numReportsUrl = 'cooked/' + metricName + '/num_reports.csv'; + ajaxGet(numReportsUrl, statusElem, function(xhr) { + var csvData = xhr.responseText; + var elem = document.getElementById('num-reports-dy'); + var g = new Dygraph(elem, csvData); + }); + + var massUrl = 'cooked/' + metricName + '/mass.csv'; + ajaxGet(massUrl, statusElem, function(xhr) { + var csvData = xhr.responseText; + var elem = document.getElementById('mass-dy'); + var g = new Dygraph(elem, csvData); + }); + + var tableUrl = 'cooked/' + metricName + '/status.part.html'; + ajaxGet(tableUrl, statusElem, function(xhr) { + var htmlData = xhr.responseText; + var elem = document.getElementById('status_table'); + elem.innerHTML = htmlData; + + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// NOTE: This was for optional Dygraphs error bars, but it's not hooked up yet. +function onMetricCheckboxClick(checkboxElem, proportionsDygraph) { + var checked = checkboxElem.checked; + if (proportionsDygraph === null) { + console.log('NULL'); + } + proportionsDygraph.updateOptions({customBars: checked}); + console.log('HANDLED'); +} + +// for day.html. +function initDay(urlHash, tableStates, statusElem) { + var jobId = urlHash.get('jobId'); + var metricName = urlHash.get('metric'); + var date = urlHash.get('date'); + + var err = ''; + if (!jobId) { + err = 'jobId missing from hash'; + } + if (!metricName) { + err = 'metric missing from hash'; + } + if (!date) { + err = 'date missing from hash'; + } + if (err) { + appendMessage(statusElem, err); + } + + // Add title and page element + var titleStr = metricName + ' on ' + date; + document.title = titleStr; + var mElem = document.getElementById('metricDay'); + mElem.innerHTML = titleStr; + + // Add correct links. + var u = document.getElementById('underlying'); + u.href = '../' + jobId + '/raw/' + metricName + '/' + date + + '/results.csv'; + + // Add correct links. + var u_res = document.getElementById('residual'); + u_res.src = '../' + jobId + '/raw/' + metricName + '/' + date + + '/residual.png'; + + var url = '../' + jobId + '/cooked/' + metricName + '/' + date + '.part.html'; + ajaxGet(url, statusElem, function(xhr) { + var htmlData = xhr.responseText; + var elem = document.getElementById('results_table'); + elem.innerHTML = htmlData; + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// for assoc-overview.html. +function initAssocOverview(urlHash, tableStates, statusElem) { + ajaxGet('cooked/assoc-overview.part.html', statusElem, function(xhr) { + var elem = document.getElementById('overview'); + elem.innerHTML = xhr.responseText; + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// for assoc-metric.html. +function initAssocMetric(urlHash, tableStates, statusElem) { + var metricName = urlHash.get('metric'); + if (metricName === undefined) { + appendMessage(statusElem, "Missing metric name in URL hash."); + return; + } + + // Add title and page element + var title = metricName + ': pairs of variables'; + document.title = title; + var pageTitleElem = document.getElementById('pageTitle'); + pageTitleElem.innerHTML = title; + + // Add correct links. + var u = document.getElementById('underlying-status'); + u.href = 'cooked/' + metricName + '/metric-status.csv'; + + var csvPath = 'cooked/' + metricName + '/metric-status.part.html'; + ajaxGet(csvPath, statusElem, function(xhr) { + var elem = document.getElementById('metric_table'); + elem.innerHTML = xhr.responseText; + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// Function to help us find the *.part.html files. +// +// NOTE: This naming convention matches the one defined in task_spec.py +// AssocTaskSpec. +function formatAssocRelPath(metricName, var1, var2) { + var varDir = var1 + '_X_' + var2.replace('..', '_'); + return metricName + '/' + varDir; +} + +// for assoc-pair.html +function initAssocPair(urlHash, tableStates, statusElem, globals) { + + var metricName = urlHash.get('metric'); + if (metricName === undefined) { + appendMessage(statusElem, "Missing metric name in URL hash."); + return; + } + var var1 = urlHash.get('var1'); + if (var1 === undefined) { + appendMessage(statusElem, "Missing var1 in URL hash."); + return; + } + var var2 = urlHash.get('var2'); + if (var2 === undefined) { + appendMessage(statusElem, "Missing var2 in URL hash."); + return; + } + + var relPath = formatAssocRelPath(metricName, var1, var2); + + // Add title and page element + var title = metricName + ': ' + var1 + ' vs. ' + var2; + document.title = title; + var pageTitleElem = document.getElementById('pageTitle'); + pageTitleElem.innerHTML = title; + + // Add correct links. + var u = document.getElementById('underlying-status'); + u.href = 'cooked/' + relPath + '/pair-status.csv'; + + /* + var distUrl = 'cooked/' + metricName + '/dist.csv'; + var u2 = document.getElementById('underlying-dist'); + u2.href = distUrl; + */ + + var tableUrl = 'cooked/' + relPath + '/pair-status.part.html'; + ajaxGet(tableUrl, statusElem, function(xhr) { + var htmlData = xhr.responseText; + var elem = document.getElementById('status_table'); + elem.innerHTML = htmlData; + + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// for assoc-day.html. +function initAssocDay(urlHash, tableStates, statusElem) { + var jobId = urlHash.get('jobId'); + var metricName = urlHash.get('metric'); + var var1 = urlHash.get('var1'); + var var2 = urlHash.get('var2'); + var date = urlHash.get('date'); + + var err = ''; + if (!jobId) { + err = 'jobId missing from hash'; + } + if (!metricName) { + err = 'metric missing from hash'; + } + if (!var1) { + err = 'var1 missing from hash'; + } + if (!var2) { + err = 'var2 missing from hash'; + } + if (!date) { + err = 'date missing from hash'; + } + if (err) { + appendMessage(statusElem, err); + } + + // Add title and page element + var titleStr = metricName + ': ' + var1 + ' vs. ' + var2 + ' on ' + date; + document.title = titleStr; + var mElem = document.getElementById('metricDay'); + mElem.innerHTML = titleStr; + + var relPath = formatAssocRelPath(metricName, var1, var2); + + // Add correct links. + var u = document.getElementById('underlying'); + u.href = '../' + jobId + '/raw/' + relPath + '/' + date + + '/assoc-results.csv'; + + var url = '../' + jobId + '/cooked/' + relPath + '/' + date + '.part.html'; + ajaxGet(url, statusElem, function(xhr) { + var htmlData = xhr.responseText; + var elem = document.getElementById('results_table'); + elem.innerHTML = htmlData; + makeTablesSortable(urlHash, [elem], tableStates); + updateTables(urlHash, tableStates, statusElem); + }); +} + +// This is the onhashchange handler of *all* HTML files. +function onHashChange(urlHash, tableStates, statusElem) { + updateTables(urlHash, tableStates, statusElem); +}
' + name + '' + meta.owners + '' + meta.summary + '