Skip to content
Permalink
Browse files

update visualizer to correctly do charts for all runs (#865)

* update visualizer to correctly do charts for all runs

* remove dead code

* comments

* typos
  • Loading branch information...
amj committed Aug 13, 2019
1 parent de866b6 commit 57970ccdca3e8a7b5cc30abde5bfbd835632eae7
Showing with 104 additions and 52 deletions.
  1. +13 −23 oneoffs/joseki.html
  2. +91 −29 oneoffs/opening_freqs_export.py
@@ -20,18 +20,11 @@
<v-container grid-list-small text-xs-left>
<v-layout row>
<v-flex s12>
<h1> Minigo Opening Analysis </h1>
{% raw %}
<v-btn-toggle v-model=run mandatory>
<v-btn v-for="(value, run) in run_times" :value=run v-on:click="this.run=run;drawChart();">
{{ run }}
</v-btn>
</v-btn-toggle>
{% endraw %}
<h1> Minigo Opening Analysis for {{run}} </h1>
</v-flex>
</v-layout>

<v-layout v-for="opening in top10" row :key="opening.sequence">
<v-layout v-for="opening in topSeq" row :key="opening.sequence + opening.run">
<v-flex xs4 lg3>
<v-card>
<v-card-text>
@@ -56,40 +49,37 @@ <h1> Minigo Opening Analysis </h1>
var app = new Vue({
el: 'main',
data: {
top10: {{giant_blob}},
topSeq: {{giant_blob}},
run: 'v17',
run_times: { v17: [1549908000.0, 1551196800.0],
v16: [1547794800.0, 1549850400.0],
}
run_times: {{time_ranges}},
},
mounted() {
google.charts.load('current', {'packages':['corechart']});
google.charts.setOnLoadCallback(this.drawChart);
},
methods: {
drawChart() {
console.log("Drawing. run:", this.run);
this.$refs.playerNodes.forEach((node, index) => {
var player = new WGo.BasicPlayer(node, {
sgf: "(;SZ[19];" + this.top10[index].sequence + ")",
sgf: "(;SZ[19];" + this.topSeq[index].sequence + ")",
layout: "minimal",
board: {section: {bottom: 9, left: 9, top:0, right: 0}},
move: 999,
});
});
this.$refs.chartNodes.forEach((node, index) => {
var data = new google.visualization.DataTable();
data.addColumn('number', 'Timestamp'); // Implicit domain label col.
data.addColumn('number', 'Frequency'); // Implicit series 1 data col.
data.addRows(this.top10[index].chart);
var data = new google.visualization.DataTable({
cols: this.topSeq[index].cols,
rows: this.topSeq[index].rows},
0.6);
var options = {
title: `Seen total ${this.top10[index].count} times`,
hAxis: {title: 'Timestamp of training',
viewWindow: {min: this.run_times[this.run][0], max: this.run_times[this.run][1] }},
title: `Seen total ${this.topSeq[index].count} times`,
hAxis: {title: '% of training',
viewWindow: {min: 0, max: 100}},
vAxis: {title: 'Frequency', logScale: true},
legend: { position : 'none'},
legend: { position : 'bottom'},
theme: 'material',
pointSize: 3,
}
@@ -21,8 +21,10 @@

import os
import sqlite3
import functools
import json
import datetime as dt
import collections
from hashlib import sha256
from jinja2 import Template

@@ -37,6 +39,7 @@
flags.DEFINE_string("db_path", 'joseki.db', "Path to josekidb")
flags.DEFINE_string("template", 'joseki.html', "Path to template to render")
flags.DEFINE_string("out_file", 'openings.html', "Where to write the report")
flags.DEFINE_integer("top_n", 100, "Number of top openings to use per run")


def sha(sequence):
@@ -167,7 +170,7 @@ def create_hourly_reports(hour_directory):
cur = db.execute('''
select seq, sum(count) from joseki_counts where hour = ? group by seq order by 2 desc limit 300;
''', (hr,))
sequences = [row for row in cur.fetchall()]
sequences = list(cur.fetchall())

dias = [make_wgo_diagram(seq, c) for (seq, c) in sequences]
with open(os.path.join(FLAGS.in_dir, '{}-out.html'.format(hr)), 'w') as out:
@@ -190,44 +193,103 @@ def create_hourly_reports(hour_directory):
"""


def create_top_report():
def get_runs(db):
return [r[0] for r in db.execute('''
select distinct(run) from joseki_counts;'''
).fetchall()]

def top_seqs_by_run(db, topN=300):
runs = get_runs(db)
top_seqs_by_run = {}
for r in runs:
cur = db.execute('''
select seq, sum(count) from joseki_counts
where run = ?
group by seq order by 2 desc limit ?;
''', (r, topN))
sequences = list(cur.fetchall())
sequences = collapse_common_prefix(sequences)
print("Pruned to ", len(sequences))
sequences.sort(key=lambda s: s[1], reverse=True)
top_seqs_by_run[r] = sequences
return top_seqs_by_run


def run_time_ranges(db):
ts = lambda hr: int(dt.datetime.strptime(hr, "%Y-%m-%d-%H").timestamp())
runs = {r[0]: (ts(r[1]), ts(r[2])) for r in db.execute('''
select run, min(hour), max(hour) from joseki_counts group by 1;
''').fetchall()}
return runs


def build_run_time_transformers(ranges, buckets=250):
""" Build a dict of functions to transform from a timestamp into a relative
offset. E.g.
input: {'v17': (1234567890, 1235567879) ... }
output: {'v17': lambda t: (t - min) * (1/max) ... }
"""

funcs = {}
def f(t, min_, max_):
#return "%0.2f" % ((t-min_) * (1/(max_-min_)))
key = (t-min_) * (1/(max_-min_))
return "%0.3f" % (int(buckets*key) / (buckets / 100.0))

for run, range_ in ranges.items():
funcs[run] = functools.partial(f, min_=range_[0], max_=range_[1])

return funcs


def create_top_report(top_n=100):
"""
Creates an html page showing the most common sequences in the database, and
charting their popularity over time.
"""
db = sqlite3.connect(FLAGS.db_path)
ts = lambda hr: int(dt.datetime.strptime(hr, "%Y-%m-%d-%H").timestamp())
print('querying')
db = sqlite3.connect(FLAGS.db_path)
cur = db.execute('''
select seq, sum(count) from joseki_counts group by seq order by 2 desc limit 300;
''')
sequences = [row for row in cur.fetchall()]

sequences = collapse_common_prefix(sequences)
sequences.sort(key=lambda s: s[1], reverse=True)
print("Pruned to ", len(sequences))

data = []
for seq, count in sequences[:30]:
print(seq, count)
cur = db.execute('''
SELECT hour, count from joseki_counts where seq = ?;
''', (seq,))

dat = cur.fetchall()
dat = [(ts(d[0]), d[1]) for d in dat]
obj = {"count": count, "chart": dat, "sequence": seq}
data.append(obj)

print('saving')
tmpl = Template(open('oneoffs/joseki.html').read())
with open(FLAGS.out_file, 'w') as out:
out.write(tmpl.render(giant_blob=json.dumps(data)))
ranges = run_time_ranges(db)
interps = build_run_time_transformers(ranges)
seqs_by_run = top_seqs_by_run(db, top_n)
runs = sorted(seqs_by_run.keys())

cols = []
cols.append({'id': 'time', 'label': '% of Training', 'type': 'number'})
for run in runs:
cols.append({'id': run + 'count', 'label': run + ' times seen', 'type': 'number'})

for run in runs:
data = []
sequences = seqs_by_run[run]
for seq, count in sequences:
print(run, seq, count)
rows = collections.defaultdict(lambda: [0 for i in range(len(runs))])

for idx, r in enumerate(runs):
cur = db.execute('''
SELECT hour, count from joseki_counts where seq = ? and run = ?;
''', (seq, r))

for hr, ct in cur.fetchall():
key = interps[r](ts(hr))
rows[key][idx] = ct

row_data = [ {'c': [ {'v': key} ] + [{'v': v if v else None} for v in value ] }
for key,value in rows.items()]
obj = {'run': run, "count": count, 'cols': cols, "rows": row_data, "sequence": seq}
data.append(obj)

print('saving')
tmpl = Template(open('oneoffs/joseki.html').read())
with open(run + FLAGS.out_file, 'w') as out:
out.write(tmpl.render(giant_blob=json.dumps(data), run=run, time_ranges=json.dumps(ranges)))


def main(_):
""" Entrypoint for absl.app """
create_top_report()
create_top_report(FLAGS.top_n)

if __name__ == '__main__':
app.run(main)

0 comments on commit 57970cc

Please sign in to comment.
You can’t perform that action at this time.