Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
1661 lines (1266 sloc) 51 KB
#
# Collective Knowledge (various graphs for experiment)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#
cfg={} # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)
# Local settings
var_post_subgraph='subgraph'
var_post_cur_subgraph='cur_subgraph'
var_post_tmp_graph_file='graph_tmp_file'
var_post_refresh_graph='refresh_graph'
var_post_reset_graph='reset_graph'
var_post_autorefresh='graph_autorefresh'
var_post_autorefresh_time='graph_autorefresh_time'
##############################################################################
# Initialize module
def init(i):
"""
Input: {}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
return {'return':0}
##############################################################################
# plot universal graph by flat dimensions
def plot(i):
"""
Input: {
(load_table_from_file) - load table directly from file
or
Select entries or table:
(repo_uoa) or (experiment_repo_uoa) - can be wild cards
(remote_repo_uoa) - if remote access, use this as a remote repo UOA
(module_uoa) or (experiment_module_uoa) - can be wild cards
(data_uoa) or (experiment_data_uoa) - can be wild cards
(repo_uoa_list) - list of repos to search
(module_uoa_list) - list of module to search
(data_uoa_list) - list of data to search
(search_dict) - search dict
(ignore_case) - if 'yes', ignore case when searching
OR
table - experiment table (if drawing from other functions)
(table_info) - point description
(flat_keys_list) - list of flat keys to extract from points into table
(order is important: for example, for plot -> X,Y,Z)
(flat_keys_list_separate_graphs) - [ [keys], [keys], ...] - several graphs ...
(sort_index) - (int) dimension to sort
(labels_for_separate_graphs) - list of labels for separate graphs
(flat_keys_index) - add all flat keys starting from this index
(flat_keys_index_end) - add all flat keys ending with this index (default #min)
(customize_dots) - if 'yes' and MPL engine, customize color and size from table_info
(out_to_file) - save picture or html to file, if supported
(will be preserved and not replotted - useful to have a copy of an original image
when replotting graphs in interactive papers)
(out_repo_uoa) - repo uoa where to save file (when reproducing graphs for interactive articles)
(out_module_uoa) - module uoa where to save file (when reproducing graphs for interactive articles)
(out_data_uoa) - data uoa where to save file (when reproducing graphs for interactive articles)
(out_id) - graph ID in the created graph entry
(out_common_meta) - graph common meta in the created graph entry
(out_graph_extra_meta) - graph extra meta (parameters and description)
(save_table_to_json_file) - save table to json file
(save_info_table_to_json_file) - save info table (mtable) to json file
(save_table_to_csv_file) - save table to csv file (need keys)
(save_to_html) - if interactive or html-based graph, save to html
(save_to_style) - if interactive or html-based graph, save to style (if needed)
(display_x_error_bar) - if 'yes', display error bar on X axis (using next dim)
(display_y_error_bar) - if 'yes', display error bar on Y axis (using next dim)
(display_z_error_bar) - if 'yes', display error bar on Z axis (using next dim)
Graphical parameters:
plot_type - mpl_2d_scatter
point_style - dict, setting point style for each separate graph {"0", "1", etc}
x_ticks_period - (int) for bar graphs, put periodicity when to show number
xmin
xmax
ymin
ymax
bound_lines - if 'yes', plot lines bounding all points ...
bound_style - ':' by default
bound_color - 'r' by default
skip_colorbar - if 'yes', do not show colorbar in heatmaps
colorbar_pad - (default = 0.15) - pad colorbar
h_lines - list of X for horizontal lines
h_lines_style
h_lines_color
v_lines - list of Y for vertical lines
v_lines_style
v_lines_color
If density graph:
(bins) - number of bins (int, default = 100)
(cov_factor) - float covariance factor
d3_div - div ID (ck_interactive). "body" if empty
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
(html) - html, if HTML generator such as d3
(style) - style for html, if HTML generator such as d3
}
"""
import os
import json
o=i.get('out','')
pst=i.get('point_style',{})
otf=i.get('out_to_file','')
otf_ruoa=i.get('out_repo_uoa','')
otf_muoa=i.get('out_module_uoa','')
otf_duoa=i.get('out_data_uoa','')
xtp=i.get('x_ticks_period','')
if xtp=='' or xtp==0: xtp=1
if xtp!='': xtp=int(xtp)
lsg=i.get('labels_for_separate_graphs',[])
stjf=i.get('save_table_to_json_file','')
sitjf=i.get('save_info_table_to_json_file','')
stcf=i.get('save_table_to_csv_file','')
table=i.get('table',[])
mtable=i.get('table_info', [])
rk=i.get('real_keys',[])
cdots=i.get('customize_dots','')
ltfj=i.get('load_table_from_file','')
if ltfj!='':
rx=ck.load_json_file({'json_file':ltfj})
if rx['return']>0: return rx
table=rx['dict']
# Check if table already there
if len(table)==0:
# Get table from entries
tmp_a=i.get('action','')
tmp_mu=i.get('module_uoa','')
i['action']='get'
i['module_uoa']=cfg['module_deps']['experiment']
if i.get('remote_repo_uoa','')!='':
i['repo_uoa']=i['remote_repo_uoa']
del(i['remote_repo_uoa'])
if 'out' in i: del(i['out'])
r=ck.access(i)
if r['return']>0: return r
table=r['table']
mtable=r.get('mtable',{})
rk=r['real_keys']
merged_meta=r.get('merged_meta',{})
pifs=r.get('plot_info_from_scenario',{})
if len(pifs)>0:
import copy
ii=copy.deepcopy(i)
i=pifs
i.update(ii)
i['action']=tmp_a
i['module_uoa']=tmp_mu
else:
# If sort/substitute
si=i.get('sort_index','')
if si!='':
rx=ck.access({'action':'sort_table',
'module_uoa':cfg['module_deps']['experiment'],
'table':table,
'sort_index':si})
if rx['return']>0: return rx
table=rx['table']
# Substitute all X with a loop (usually to sort Y and compare with predictions in scatter graphs, etc)
if i.get('substitute_x_with_loop','')=='yes':
rx=ck.access({'action':'substitute_x_with_loop',
'module_uoa':cfg['module_deps']['experiment'],
'table':table})
if rx['return']>0: return rx
table=rx['table']
if len(table)==0:
return {'return':1, 'error':'no points found'}
# Check if out to module
pp=''
if otf_duoa!='':
import os
if otf_muoa=='': otf_muoa=work['self_module_uid']
# Check extra params
o_id=os.path.splitext(os.path.basename(otf))[0]
o_cm=i.get('out_common_meta','')
o_gem=i.get('out_graph_extra_meta',{})
# Try to load first
ddd={}
xid=-1
ii={'action':'load',
'module_uoa':otf_muoa,
'repo_uoa':otf_ruoa,
'data_uoa':otf_duoa}
rx=ck.access(ii)
if rx['return']==0:
ddd=rx['dict']
dddx=ddd.get('graphs',[])
if o_id!='':
for q in range(0, len(dddx)):
if dddx[q].get('id','')==o_id:
xid=q
break
# Updating
if 'graphs' not in ddd:
ddd['graphs']=[]
ddd.update(o_cm)
# Prepare graph params
import copy
ii=copy.deepcopy(i)
for q in cfg['remove_keys_for_interactive_graphs']:
if q in ii: del(ii[q])
dddg={'params':ii}
dddg.update(o_gem)
if o_id!='':
dddg['id']=o_id
else:
dddg['id']='default'
if len(i.get('out_graph_meta',{}))>0:
dddg.update(i['out_graph_meta'])
if xid!=-1:
ddd['graphs'][xid]=dddg
else:
ddd['graphs'].append(dddg)
# Try to update this entry to be sure that we can record there, and get path
ii={'action':'update',
'module_uoa':otf_muoa,
'repo_uoa':otf_ruoa,
'data_uoa':otf_duoa,
'dict':ddd,
'substitute':'yes',
'ignore_update':'yes'}
rx=ck.access(ii)
if rx['return']>0: return rx
pp=rx['path']
# Save table to JSON file, if needed
if stjf!='':
if pp!='':
ppx=os.path.join(pp, stjf)
else:
ppx=stjf
rx=ck.save_json_to_file({'json_file':ppx, 'dict':table})
if rx['return']>0: return rx
# Save info table to JSON file, if needed
if sitjf!='':
if pp!='':
ppx=os.path.join(pp, sitjf)
else:
ppx=sitjf
rx=ck.save_json_to_file({'json_file':ppx, 'dict':mtable})
if rx['return']>0: return rx
# Save table to CSV file, if needed
if stcf!='':
if pp!='':
ppx=os.path.join(pp, stcf)
else:
ppx=stcf
ii={'action':'convert_table_to_csv',
'module_uoa':cfg['module_deps']['experiment'],
'table':table,
'keys':rk,
'merge_multi_tables':'yes',
'file_name':ppx}
rx=ck.access(ii)
if rx['return']>0: return rx
# Prepare libraries
pt=i.get('plot_type','')
html=''
style=''
hlines=i.get('h_lines',[])
vlines=i.get('v_lines',[])
# Check if display error bars
xerr=i.get('display_x_error_bar','')
yerr=i.get('display_y_error_bar','')
zerr=i.get('display_z_error_bar','')
xerr2=i.get('display_x_error_bar2','') # type 2
yerr2=i.get('display_y_error_bar2','')
zerr2=i.get('display_z_error_bar2','')
# Find min/max in all data and all dimensions / per sub-graph
tmin=[]
tmax=[]
stmin={}
stmax={}
for g in table:
gt=table[g]
stmin[g]=[]
stmax[g]=[]
mgt=[]
if g in mtable:
mgt=mtable[g]
xpst=pst.get(g,{})
remove_permanent=False
if xpst.get('remove_permanent','')=='yes':
remove_permanent=True
leave_only_permanent=False
if xpst.get('leave_only_permanent','')=='yes':
leave_only_permanent=True
ngt=[]
# for k in gt:
for uindex in range(0,len(gt)):
k=gt[uindex]
if (remove_permanent or leave_only_permanent) and uindex<len(mgt):
mu=mgt[uindex]
if remove_permanent and mu.get('permanent','')=='yes':
continue
if leave_only_permanent and mu.get('permanent','')!='yes':
continue
ngt.append(k)
if xpst.get('skip_from_dims','')=='yes':
continue
for d in range(0, len(k)):
v=k[d]
if len(tmin)<=d:
tmin.append(v)
tmax.append(v)
else:
if v!=None and v<tmin[d]: tmin[d]=v
if v!=None and v>tmax[d]: tmax[d]=v
if len(stmin[g])<=d and v!=None:
stmin[g].append(v)
stmax[g].append(v)
else:
if v!=None and v<stmin[g][d]: stmin[g][d]=v
if v!=None and v>stmax[g][d]: stmax[g][d]=v
table[g]=ngt
####################################################################### MPL ###
if pt.startswith('mpl_'):
import matplotlib as mpl
if ck.cfg.get('use_internal_engine_for_plotting','')=='yes':
mpl.use('agg') # if XWindows is not installed, use internal engine
elif os.environ.get('CK_MPL_BACKEND','')!='':
mpl.use(os.environ['CK_MPL_BACKEND'])
import matplotlib.pyplot as plt
fsize=int(i.get('font_size','10'))
fweight=i.get('font_weight','normal')
ffamily=i.get('font_family','arial')
# Set font
font=i.get('font',{})
if len(font)==0:
font = {'family':ffamily,
'weight':fweight,
'size': fsize}
plt.rc('font', **font)
# Configure graph
gs=cfg['mpl_point_styles']
sizex=i.get('mpl_image_size_x','')
if sizex=='': sizex='9'
sizey=i.get('mpl_image_size_y','')
if sizey=='': sizey='5'
dpi=i.get('mpl_image_dpi','')
if dpi=='': dpi='100'
if sizex!='' and sizey!='' and dpi!='':
fig=plt.figure(figsize=(int(sizex),int(sizey)))
else:
fig=plt.figure()
if i.get('plot_grid','')=='yes':
plt.grid(True)
bl=i.get('bound_lines','')
if pt=='mpl_3d_scatter' or pt=='mpl_3d_trisurf':
from mpl_toolkits.mplot3d import Axes3D
sp=fig.add_subplot(111, projection='3d')
else:
sp=fig.add_subplot(111)
if i.get('xscale_log','')=='yes': sp.set_xscale('log')
if i.get('yscale_log','')=='yes': sp.set_yscale('log')
if i.get('zscale_log','')=='yes': sp.set_zscale('log')
# If density or heatmap, find min and max for both graphs:
if pt=='mpl_1d_density' or pt=='mpl_1d_histogram' or pt=='mpl_2d_heatmap' or pt=='mpl_3d_scatter' or pt=='mpl_3d_trisurf':
index=0
if pt=='mpl_2d_heatmap': index=2
dmean=0.0
start=True
dmin=0.0
dmax=0.0
it=0
dt=0
for g in table:
gt=table[g]
for k in gt:
v=k[index]
if v!=None and v!='':
if start:
dmin=v
start=False
else:
dmin=min(dmin, v)
if start:
dmax=v
start=False
else:
dmax=max(dmax, v)
it+=1
dt+=v
if it!=0: dmean=dt/it
# If heatmap, prepare colorbar
if pt=='mpl_2d_heatmap' or pt=='mpl_3d_trisurf':
from matplotlib import cm
if len(i.get('color_dict',{}))>0:
xcmap = mpl.colors.LinearSegmentedColormap('my_colormap', i['color_dict'], 1024)
else:
xcmap = plt.cm.get_cmap('coolwarm')
if i.get('shifted_colormap','')=='yes':
r=ck.load_module_from_path({'path':work['path'], 'module_code_name':'module_shifted_colormap', 'skip_init':'yes'})
if r['return']>0: return r
scm=r['code']
xx_start=dmin
if i.get('shifted_colormap_start','')!='':
xx_start=i['shifted_colormap_start']
xx_stop=dmax
if i.get('shifted_colormap_stop','')!='':
xx_stop=i['shifted_colormap_stop']
xx_mid=1.0
if i.get('shifted_colormap_mid','')!='':
xx_mix=i['shifted_colormap_mid']
xcmap = scm.shiftedColorMap(xcmap, start=xx_start, stop=xx_stop, midpoint=xx_mid, name='shifted')
# Check forced min/max for different axis
xmin=i.get('xmin','')
xmax=i.get('xmax','')
ymin=i.get('ymin','')
ymax=i.get('ymax','')
zmin=i.get('zmin','')
zmax=i.get('zmax','')
if xmin!='':
sp.set_xlim(left=float(xmin))
if xmax!='':
sp.set_xlim(right=float(xmax))
if ymin!='':
sp.set_ylim(bottom=float(ymin))
if ymax!='':
sp.set_ylim(top=float(ymax))
if zmin!='':
sp.set_zlim(bottom=float(zmin))
if zmax!='':
sp.set_zlim(top=float(zmax))
# Check if invert axis
if i.get('invert_x_axis','')=='yes': plt.gca().invert_xaxis()
if i.get('invert_y_axis','')=='yes': plt.gca().invert_yaxis()
if i.get('invert_z_axis','')=='yes': plt.gca().invert_zaxis()
if pt=='mpl_2d_bars' or pt=='mpl_2d_lines':
ind=[]
gt=table['0']
xt=0
for q in gt:
xt+=1
if xt==xtp:
v=q[0]
xt=0
else:
v=0
ind.append(v)
sp.set_xticks(ind)
sp.set_xticklabels(ind, rotation=-20)
width=0.9/len(table)
# Iterate over separate graphs and add points
s=0
for g in sorted(table, key=int):
gt=table[g]
mgt=[]
if g in mtable:
mgt=mtable[g]
lbl=''
if s<len(lsg): lbl=lsg[s]
xpst=pst.get(g,{})
elw=int(xpst.get('elinewidth',0))
xfmt=xpst.get('fmt','')
xcapsize=xpst.get('capsize','')
if xcapsize=='' or xcapsize==None: xcapsize=None
else: xcapsize=int(xcapsize)
cl=xpst.get('color','')
if cl=='': cl=gs[s]['color']
sz=xpst.get('size','')
if sz=='': sz=gs[s]['size']
connect_lines=xpst.get('connect_lines','')
mrk=xpst.get('marker','')
if mrk=='': mrk=gs[s]['marker']
lst=xpst.get('line_style','')
if lst=='': lst=gs[s].get('line_style', '-')
heatmap=None
if pt=='mpl_2d_scatter' or pt=='mpl_2d_bars' or pt=='mpl_2d_lines':
mx=[]
mxerr=[]
my=[]
myerr=[]
mcolor=[]
msize=[]
# for u in gt:
for uindex in range(0,len(gt)):
u=gt[uindex]
iu=0
# Check if extra info (color and size)
minfo={}
if uindex<len(mgt):
minfo=mgt[uindex]
xcl=cl
if minfo.get('color','')!='':
xcl=minfo['color']
mcolor.append(xcl)
xsz=sz
if minfo.get('size','')!='':
xsz=minfo['size']
msize.append(int(xsz))
# Check if no None
partial=False
for q in u:
if q==None:
partial=True
break
if not partial:
mx.append(u[iu])
iu+=1
if xerr=='yes':
mxerr.append(u[iu])
iu+=1
my.append(u[iu])
iu+=1
if yerr=='yes':
myerr.append(u[iu])
iu+=1
if pt=='mpl_2d_bars':
mx1=[]
# mx2=[]
# names={}
# iq=0
for q in mx:
# if type(q)!=int and type(q)!=float:
# if q in names: q=names[q]
# else:
# names[q]=iq
# q=iq
# iq+=1
# mx2.append(str(q))
mx1.append(q+width*s)
if yerr=='yes':
sp.bar(mx1, my, width=width, edgecolor=cl, facecolor=cl, align='center', yerr=myerr, label=lbl) # , error_kw=dict(lw=2))
else:
sp.bar(mx1, my, width=width, edgecolor=cl, facecolor=cl, align='center', label=lbl)
elif pt=='mpl_2d_lines':
if yerr=='yes':
sp.errorbar(mx, my, yerr=myerr, ls='none', c=cl, elinewidth=elw)
sp.plot(mx, my, c=cl, label=lbl)
else:
draw_scatter=False
if xerr=='yes' and yerr=='yes':
if cdots=='yes':
sp.errorbar(mx, my, xerr=mxerr, yerr=myerr, ls='none', c=mcolor, elinewidth=elw, label=lbl, fmt=xfmt)
else:
sp.errorbar(mx, my, xerr=mxerr, yerr=myerr, ls='none', c=cl, elinewidth=elw, label=lbl, fmt=xfmt)
elif xerr=='yes' and yerr!='yes':
if cdots=='yes':
sp.errorbar(mx, my, xerr=mxerr, ls='none', c=mcolor, elinewidth=elw, label=lbl, fmt=xfmt)
else:
sp.errorbar(mx, my, xerr=mxerr, ls='none', c=cl, elinewidth=elw, label=lbl, fmt=xfmt)
elif yerr=='yes' and xerr!='yes':
if cdots=='yes':
sp.errorbar(mx, my, yerr=myerr, ls='none', c=mcolor, elinewidth=elw, label=lbl, fmt=xfmt, capsize=xcapsize)
else:
sp.errorbar(mx, my, yerr=myerr, ls='none', c=cl, elinewidth=elw, label=lbl, fmt=xfmt, capsize=xcapsize)
else:
draw_scatter=True
if draw_scatter or i.get('force_center_dot','')=='yes':
if cdots=='yes':
sp.scatter(mx, my, s=msize, edgecolor=mcolor, c=mcolor, marker=mrk, label=lbl)
else:
sp.scatter(mx, my, s=int(sz), edgecolor=cl, c=cl, marker=mrk, label=lbl)
if connect_lines=='yes':
if cdots=='yes':
sp.plot(mx, my, c=mcolor, label=lbl)
else:
sp.plot(mx, my, c=cl, label=lbl)
if xpst.get('frontier','')=='yes':
# not optimal solution, but should work (need to sort to draw proper frontier)
a=[]
for q in range(0, len(mx)):
a.append([mx[q],my[q]])
b=sorted(a, key=lambda k: k[0])
if xpst.get('reuse_dims','')=='yes':
mx=[b[0][0]]
my=[tmax[1]]
elif xpst.get('dims_from_this_graph','')=='yes':
mx=[stmin[g][0]]
my=[stmax[g][1]]
else:
mx=[tmin[0]]
my=[tmax[1]]
for j in b:
mx.append(j[0])
my.append(j[1])
if xpst.get('reuse_dims','')=='yes':
mx.append(tmax[0])
my.append(b[-1][1])
elif xpst.get('dims_from_this_graph','')=='yes':
mx.append(stmax[g][0])
my.append(stmin[g][1])
else:
mx.append(tmax[0])
my.append(tmin[1])
sp.plot(mx, my, c=cl, linestyle=lst, label=lbl)
elif pt=='mpl_1d_density' or pt=='mpl_1d_histogram':
if not start: # I.e. we got non empty points
xbins=i.get('bins', 100)
xcov_factor=i.get('cov_factor', '')
mx=[]
for u in gt:
mx.append(u[0])
ii={'action':'analyze',
'min':dmin,
'max':dmax,
'module_uoa':cfg['module_deps']['math.variation'],
'bins':xbins,
'cov_factor':xcov_factor,
'characteristics_table':mx}
r=ck.access(ii)
if r['return']>0: return r
xs=r['xlist']
dxs=r['ylist']
pxs=r['xlist2s']
dpxs=r['ylist2s']
if pt=='mpl_1d_density':
sp.plot(xs,dxs, label=lbl)
sp.plot(pxs, dpxs, 'x', mec='r', mew=2, ms=8) #, mfc=None, mec='r', mew=2, ms=8)
sp.plot([dmean,dmean],[0,dpxs[0]],'g--',lw=2)
else:
plt.hist(mx, bins=xbins, normed=True, label=lbl)
elif pt=='mpl_2d_heatmap' or pt=='mpl_3d_scatter' or pt=='mpl_3d_trisurf':
mx=[]
mxerr=[]
my=[]
myerr=[]
mz=[]
mzerr=[]
for u in gt:
iu=0
# Check if no None
partial=False
for q in u:
if q==None:
partial=True
break
if not partial:
mx.append(u[iu])
iu+=1
if xerr=='yes':
mxerr.append(u[iu])
iu+=1
my.append(u[iu])
iu+=1
if yerr=='yes':
myerr.append(u[iu])
iu+=1
mz.append(u[iu])
iu+=1
if zerr=='yes':
mzerr.append(u[iu])
iu+=1
if pt=='mpl_2d_heatmap':
heatmap=sp.scatter(mx, my, c=mz, s=int(sz), marker=mrk, lw=elw, cmap=xcmap)
elif pt=='mpl_3d_scatter':
heatmap=sp.scatter(mx,my,mz, c=cl, s=int(sz), marker=mrk, lw=elw)
elif pt=='mpl_3d_trisurf':
# heatmap=sp.plot_trisurf(mx,my,mz,cmap=cm.coolwarm, lw=elw)
heatmap=sp.plot_trisurf(mx,my,mz,cmap=xcmap, lw=elw)
s+=1
if s>=len(gs):s=0
# If heatmap, finish colors
if (pt=='mpl_2d_heatmap' or pt=='mpl_3d_trisurf') and i.get('skip_colorbar','')!='yes':
colorbar_pad=i.get('colorbar_pad','')
if colorbar_pad=='': colorbar_pad=0.15
colorbar_pad=float(colorbar_pad)
plt.colorbar(heatmap, orientation=xpst.get('colorbar_orietation','horizontal'), label=xpst.get('colorbar_label',''), pad=colorbar_pad)
# For lines
dxmin=tmin[0]
if xmin!='': dxmin=float(xmin)
dxmax=tmax[0]
if xmax!='': dxmax=float(xmax)
dymin=tmin[1]
if ymin!='': dymin=float(ymin)
dymax=tmax[1]
if ymax!='': dymax=float(ymax)
# If bounds
if bl=='yes':
xbs=i.get('bound_style',':')
xbc=i.get('bound_color','r')
sp.plot([tmin[0],tmax[0]],[tmin[1],tmin[1]], linestyle=xbs, c=xbc)
sp.plot([tmin[0],tmin[0]],[tmin[1],tmax[1]], linestyle=xbs, c=xbc)
# If horizontal lines
if len(hlines)>0:
xbs=i.get('h_lines_style','--')
xbc=i.get('h_lines_color','r')
for q in hlines:
sp.plot([dxmin,dxmax],[q,q], linestyle=xbs, c=xbc)
# If horizontal lines
if len(vlines)>0:
xbs=i.get('v_lines_style','--')
xbc=i.get('v_lines_color','r')
for q in vlines:
sp.plot([q,q],[dymin,dymax], linestyle=xbs, c=xbc)
# Checking scaling
if i.get('x_ticks_scale','')!='':
xticks = sp.get_xticks()/float(i['x_ticks_scale'])
sp.set_xticklabels(xticks)
if i.get('y_ticks_scale','')!='':
yticks = sp.get_yticks()/float(i['y_ticks_scale'])
sp.set_yticklabels(yticks)
if i.get('z_ticks_scale','')!='':
zticks = sp.get_zticks()/float(i['z_ticks_scale'])
sp.set_zticklabels(zticks)
# Set axes labels
xlab=i.get('axis_x_labels',[])
if len(xlab)>0:
ind=[]
qq=0
for q in xlab:
ind.append(qq)
qq+=1
sp.set_xticks(ind)
xrot=i.get('axis_x_rotation','')
if xrot=='': sp.set_xticklabels(xlab)
else: sp.set_xticklabels(xlab, rotation=xrot)
ylab=i.get('axis_y_labels',[])
if len(ylab)>0:
sp.set_yticks(ylab)
sp.set_yticklabels(ylab)
# Set axes names
axd=i.get('axis_x_desc','')
if axd!='': plt.xlabel(axd)
ayd=i.get('axis_y_desc','')
if ayd!='': plt.ylabel(ayd)
atitle=i.get('title','')
if atitle!='': plt.title(atitle)
# handles, labels = plt.get_legend_handles_labels()
plt.legend() #handles, labels)
try:
plt.tight_layout()
except Exception:
pass
if otf=='':
plt.show()
else:
if pp!='':
ppx=os.path.join(pp, otf)
else:
ppx=otf
plt.savefig(ppx)
####################################################################### D3 ###
elif pt.startswith('d3_'):
# Try to load template
ppx=os.path.join(work['path'],'templates',pt+'.html')
if not os.path.isfile(ppx):
return {'return':1, 'error':'template for this graph is not found'}
rx=ck.load_text_file({'text_file':ppx})
if rx['return']>0: return rx
html=rx['string']
# Check if style is there (optional)
ppx=os.path.join(work['path'],'templates',pt+'.style')
if os.path.isfile(ppx):
rx=ck.load_text_file({'text_file':ppx})
if rx['return']>0: return rx
style=rx['string']
# Convert data table into JSON
rx=ck.dumps_json({'dict':table})
if rx['return']>0: return rx
stable=rx['string']
# Convert info table into JSON
rx=ck.dumps_json({'dict':mtable})
if rx['return']>0: return rx
smtable=rx['string']
# Convert point styles into JSON
rx=ck.dumps_json({'dict':pst})
if rx['return']>0: return rx
spst=rx['string']
html=html.replace('$#x_ticks_period#$',str(xtp))
html=html.replace('$#display_x_error_bar#$',xerr)
html=html.replace('$#display_y_error_bar#$',yerr)
html=html.replace('$#display_z_error_bar#$',zerr)
html=html.replace('$#display_x_error_bar2#$',xerr2)
html=html.replace('$#display_y_error_bar2#$',yerr2)
html=html.replace('$#display_z_error_bar2#$',zerr2)
html=html.replace('$#cm_info_json#$',smtable)
html=html.replace('$#cm_point_style_json#$',spst)
html=html.replace('$#cm_data_json#$',stable)
html=html.replace('$#cm_info_json#$',smtable)
html=html.replace('$#cm_point_style_json#$',spst)
html=html.replace('$#h_lines#$',json.dumps(hlines))
html=html.replace('$#v_lines#$',json.dumps(vlines))
# Set axes names
axd=i.get('axis_x_desc','')
html=html.replace('$#axis_x_desc#$', axd)
ayd=i.get('axis_y_desc','')
html=html.replace('$#axis_y_desc#$', ayd)
size_x=i.get('image_width','')
if size_x=='': size_x=600
html=html.replace('$#ck_image_width#$', str(size_x))
size_y=i.get('image_height','')
if size_y=='': size_y=400
html=html.replace('$#ck_image_height#$', str(size_y))
xmin=i.get('xmin','')
html=html.replace('$#ck_xmin#$', str(xmin))
xmax=i.get('xmax','')
html=html.replace('$#ck_xmax#$', str(xmax))
ymin=i.get('ymin','')
html=html.replace('$#ck_ymin#$', str(ymin))
ymax=i.get('ymax','')
html=html.replace('$#ck_ymax#$', str(ymax))
# Save html to file (do not hardwire URLs)
x=i.get('out_to_file','')
if x!='':
if pp!='':
ppx=os.path.join(pp, x)
else:
ppx=x
rx=ck.save_text_file({'text_file':ppx, 'string':html})
if rx['return']>0: return rx
# Save style to file, if needed
x=i.get('save_to_style','')
if x!='':
if pp!='':
ppx=os.path.join(pp, x)
else:
ppx=x
rx=ck.save_text_file({'text_file':ppx, 'string':style})
if rx['return']>0: return rx
# Update URLs if needed (for example, to load .js files from CK repo)
url0=i.get('wfe_url','')
if url0=='': url0=ck.cfg.get('wfe_url_prefix','')
html=html.replace('$#ck_root_url#$', url0)
# Save working html locally to visualize without CK
y=i.get('d3_div','')
y1=''
y2=''
if y=='':
y='body'
else:
y1='<div id="'+y+'">\n\n'
y2='\n</div>\n'
y='div#'+y
html=html.replace('$#ck_where#$',y)
if i.get('save_to_html','')!='':
x='<html>\n\n<style>\n'+style+'</style>\n\n'+'<body>\n\n'+y1+html+y2+'\n\n</body>\n</html>\n'
x=x.replace('$#ck_where#$',y)
rx=ck.save_text_file({'text_file':i['save_to_html'], 'string':x})
if rx['return']>0: return rx
else:
return {'return':1, 'error':'this type of plot ('+pt+') is not supported'}
return {'return':0, 'html':html, 'style':style}
##############################################################################
# Continuously updated plot
def continuous_plot(i):
"""
Input: {
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
for q in range(0, 1000):
r=plot(i)
if r['return']>0: return r
x=ck.inp({'text':'Press any key'})
return {'return':0}
##############################################################################
# view entry as html
def html_viewer(i):
"""
Input: {
data_uoa
url_base
url_pull
url_cid
(subgraph)
url_pull_tmp
tmp_data_uoa
url_wiki
html_share
form_name - current form name
(all_params)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
h=''
st=''
raw='no'
top='no'
duoa=i['data_uoa']
burl=i['url_base']
purl=i['url_pull']
wurl=i.get('url_wiki','')
url_cid=i.get('url_cid','')
tpurl=i['url_pull_tmp']
tpuoa=i['tmp_data_uoa']
ap=i.get('all_params',{})
ruoa=ap.get('ck_top_repo','')
muoa=ap.get('ck_top_module','')
if muoa=='': muoa=work['self_module_uid']
cparams=ap.get('graph_params','') # current graph params
hshare=i.get('html_share','')
itype='png'
# Check autorefresh
ar=ap.get(var_post_autorefresh,'')
if ar=='on':
ap[var_post_refresh_graph]='yes'
form_name=i['form_name']
form_submit='document.'+form_name+'.submit();'
art=ap.get(var_post_autorefresh_time,'')
iart=5
if art!='':
try:
iart=int(art)
except ValueError:
iart=5
if ar=='on':
h+='\n'
h+='<script language="javascript">\n'
h+=' <!--\n'
h+=' setTimeout(\''+form_submit+'\','+str(iart*1000)+');\n'
h+=' //-->\n'
h+='</script>\n'
h+='\n'
# Set replotting
jj={'action':'create_input',
'module_uoa':cfg['module_deps']['wfe'],
'type':'hidden',
'name': var_post_refresh_graph,
'value':'yes'}
rx=ck.access(jj)
if rx['return']>0: return rx
h+=rx['html']+'\n'
if duoa!='':
# Load entry
rx=ck.access({'action':'load',
'module_uoa':muoa,
'data_uoa':duoa})
if rx['return']>0: return rx
pp=rx['path']
dd=rx['dict']
duid=rx['data_uid']
name=dd.get('name','')
gsr=dd.get('get_shared_repo','')
sruoa=dd.get('scripts_repo_uoa','')
smuoa=dd.get('scripts_module_uoa','')
sduoa=dd.get('scripts_data_uoa','')
h+=' <span id="ck_entries1a">'+name+'</span><br>\n'
h+=' <div id="ck_entries_space4"></div>\n'
graphs=dd.get('graphs',[])
# If more than one subgraph, prepare selector
hsb=''
igraph=0
cgraph=0
x=ap.get(var_post_cur_subgraph,'')
try:
cgraph=int(x)
except ValueError:
cgraph=0
# sgraph=i.get(var_post_subgraph,'')
# if sgraph=='':
sgraph=ap.get(var_post_subgraph,'')
if sgraph=='': sgraph=i.get('subgraph','')
if len(graphs)>1:
dx=[]
jgraph=0
for q in graphs:
vid=q.get('id','')
if sgraph=='': sgraph=vid
if vid==sgraph:
igraph=jgraph
x=q.get('name','')
if x=='': x=vid
dx.append({'name':vid, 'value':vid})
jgraph+=1
jj={'action':'create_selector',
'module_uoa':cfg['module_deps']['wfe'],
'name': var_post_subgraph,
'onchange':form_submit,
'data':dx,
'selected_value':sgraph}
rx=ck.access(jj)
if rx['return']>0: return rx
hsb=rx['html']+'\n'
if igraph!=cgraph:
ap[var_post_reset_graph]='yes'
cgraph=igraph
# Save current subgraph to detect change and reset ...
jj={'action':'create_input',
'module_uoa':cfg['module_deps']['wfe'],
'type':'hidden',
'name': var_post_cur_subgraph,
'value':str(cgraph)}
rx=ck.access(jj)
if rx['return']>0: return rx
h+=rx['html']+'\n'
# Visualize
gid=''
if igraph<len(graphs):
g=graphs[igraph]
output=g.get('output','')
gid=g.get('id','')
if gid!='':
# Get graph params
if g.get('notes','')!='':
h+='<i>'+g['notes']+'</i>'
pjson=os.path.join(pp, gid+'.json')
if not os.path.isfile(pjson): pjson=''
pcsv=os.path.join(pp, gid+'.csv')
if not os.path.isfile(pcsv): pcsv=''
h+='<div id="ck_entries_space4"></div>\n'
if hshare!='':
h+='<div id="ck_entries_space4"></div>\n'
h+=hshare
h+=' <div id="ck_entries_space4"></div>\n'
h+='<div style="text-align: right;">'
if wurl!='':
h+='[&nbsp;<a href="'+wurl+'">Discussion wiki (comments, reproducibility, etc.)</a>&nbsp;]'
h+='</div>\n'
# h+=' <hr class="ck_hr">\n'
if hsb!='':
h+=' <div id="ck_entries_space4"></div>\n'
h+='<center>Select subgraph:&nbsp;'+hsb+'</center>\n'
# h+=' <hr class="ck_hr">\n'
# Check if interactive (html + style already prepared)
if output=='html':
image=gid+'.html'
else:
image=gid+'.'+itype
params=g.get('params',{})
problem_converting_json=''
if var_post_reset_graph not in ap and cparams!='':
rx=ck.convert_json_str_to_dict({'str':cparams, 'skip_quote_replacement':'yes'})
if rx['return']>0:
problem_converting_json=rx['error']
else:
params=rx['dict']
rx=ck.dumps_json({'dict':params, 'sort_keys':'yes'})
if rx['return']>0: return rx
jparams=rx['string']
# Check if need to regenerate
problem=''
image_orig=image
himage=''
if var_post_refresh_graph in ap:
image_orig=''
import copy
ii=copy.deepcopy(params)
ii['action']='plot'
ii['module_uoa']=work['self_module_uoa']
image=ap.get(var_post_tmp_graph_file,'')
if image=='':
rx=ck.gen_tmp_file({'prefix':'tmp-', 'suffix':'.'+itype, 'remove_dir':'yes'})
if rx['return']>0: return rx
image=rx['file_name']
ii['out_to_file']=image
# Preset current entry params
jj={'action':'create_input',
'module_uoa':cfg['module_deps']['wfe'],
'type':'hidden',
'name': var_post_tmp_graph_file,
'value':image}
rx=ck.access(jj)
if rx['return']>0: return rx
h+=rx['html']+'\n'
if ck.cfg.get('graph_tmp_repo_uoa','')!='':
ii['out_repo_uoa']=ck.cfg['graph_tmp_repo_uoa']
ii['out_module_uoa']='tmp'
ii['out_data_uoa']=tpuoa
# (save_table_to_json_file) - save table to json file
rx=ck.access(ii)
if rx['return']>0:
problem=rx['error']
purl=tpurl
# Prepare html
size_x=params.get('size_x','')
size_y=params.get('size_y','')
h+=' <table border="0" cellpadding="3" width="100%">\n'
h+=' <tr>\n'
extra=''
if size_x!='': extra+='width="'+str(size_x)+'" '
h+=' <td valign="top" '+extra+'>\n'
h+=' <div id="ck_entries">\n'
h+=' <b><small>Graph:</small></b>\n'
if problem!='':
h+='<br><br><span style="color:red;"><i>Problem: '+problem+'!</i></span><br>\n'
else:
if output=='html' and image!='':
# Check if style exists
pstyle=os.path.join(pp, gid+'.style')
if os.path.isfile(pstyle):
rx=ck.load_text_file({'text_file':pstyle})
if rx['return']>0: return rx
st=rx['string']
# Generate UID
rx=ck.gen_uid({})
if rx['return']>0: return rx
uid=rx['data_uid']
div_with_uid='ck_interactive_'+uid
h+='<div id="'+div_with_uid+'">\n'
z=muoa+':'+duoa
if var_post_refresh_graph in ap:
z=url_cid
import os
image_st=os.path.splitext(image)[0]+'.style'
h+='$#ck_include_start#${"cid":"'+z+'", "where":"div#'+div_with_uid+'", "html":"'+image+'", "style":"'+image_st+'"}$#ck_include_stop#$\n'
h+='</div>\n'
else:
if image!='':
if size_y!='': extra+='height="'+str(size_y)+'" '
himage='<img src="'+purl+image+'" '+extra+'>'
h+=' '+himage
h+=' </div>\n'
h+=' </td>\n'
h+=' <td valign="top">\n'
x='width:99%;'
if size_y!='': x+='height:'+str(size_y)+'px;'
h+=' <div id="ck_entries">\n'
h+=' <b><small>Graph params (to customize/reproduce):</small></b>\n'
if problem_converting_json!='':
h+='<br><br><span style="color:red;"><i>'+problem_converting_json+'</i></span><br>\n'
h+=' <textarea name="graph_params" style="'+x+'">\n'
h+=jparams+'\n'
h+=' </textarea><br>\n'
h+=' </div>\n'
h+=' </td>\n'
h+=' </tr>\n'
h+='</table>\n'
# h+=' <hr class="ck_hr">\n'
if g.get('skip_control','')!='yes':
h+='<center>\n'
h+='<button type="submit" name="'+var_post_refresh_graph+'">Replot graph</button>\n'
h+='<button type="submit" name="'+var_post_reset_graph+'">Reset graph</button>\n'
checked=''
if ar=='on': checked=' checked '
h+='&nbsp;&nbsp;&nbsp;Auto-replot graph:&nbsp;<input type="checkbox" name="'+var_post_autorefresh+'" id="'+var_post_autorefresh+'" onchange="submit()"'+checked+'>,'
h+='&nbsp;seconds: <input type="text" name="'+var_post_autorefresh_time+'" value="'+str(iart)+'">\n'
h+='</center>\n'
if g.get('skip_reproduce','')!='yes':
# h+='<hr class="ck_hr">\n'
h+='<center>\n'
h+='<div id="ck_entries" style="background-color: #dfffbf;">\n'
h+='<b>Reproducing graph:</b>\n'
h+='<table width="99%">\n'
h+=' <tr>\n'
h+=' <td valign="top" align="left" width="44%">\n'
h+=' <table border="0" cellpadding="5">\n'
h+=' <tr>\n'
h+=' <td valign="top" width="140"><b>Experiment entries:</b></td>\n'
h+=' <td valign="top"><i>\n'
duoal=params.get('data_uoa_list','')
if len(duoal)>0:
h+='\n'
for q in duoal:
h+='<a href="'+burl+'wcid=experiment:'+q+'">'+q+'</a><br>\n'
h+='\n'
h+=' </i></td>\n'
h+=' </tr>\n'
if smuoa!='' and sduoa!='':
cid=smuoa+':'+sduoa
if sruoa!='': cid=sruoa+':'+cid
h+=' <tr>\n'
h+=' <td valign="top"><b>Scripts to rebuild:</b></td>\n'
h+=' <td valign="top"><i>\n'
h+=' ck find '+cid+'<br>\n'
h+=' <a href="'+burl+'wcid='+cid+'">View in CK viewer</a>\n'
h+=' </i></td>\n'
h+=' </tr>\n'
if output!='html':
h+=' <tr>\n'
h+=' <td valign="top"><b>Replay graph from CMD:</b></td>\n'
h+=' <td valign="top"><i>\n'
h+=' ck replay graph:'+duoa+' id='+gid+'\n'
h+=' </i></td>\n'
h+=' </tr>\n'
h+=' </table>\n'
h+=' </td>\n'
h+=' <td valign="top" align="left" width="56%">\n'
h+=' <table border="0" cellpadding="5">\n'
if gsr!='':
h+=' <tr>\n'
h+=' <td valign="top" width="250"><b>Obtain shared CK repo with all artifacts:</b></td>\n'
h+=' <td valign="top">\n'
h+=' <i>'+gsr+'</i>\n'
h+=' </td>\n'
h+=' </tr>\n'
refresh=False
if var_post_refresh_graph in ap: refresh=True
# if (pjson!='' or pcsv!='') and image_orig!='' and himage!='':
# if (pjson!='' or pcsv!='') and not refresh: # if refresh, table may change
x1=purl+gid+'.json'
x2=purl+gid+'.csv'
h+=' <tr>\n'
h+=' <td valign="top"><b>Original experiment table:</b></td>\n'
h+=' <td valign="top"><i>\n'
if pjson!='':
h+=' <a href="'+x1+'">Download in JSON</a>;&nbsp;&nbsp'
if pcsv!='':
h+=' <a href="'+x2+'">Download in CSV</a>\n'
h+=' </i></td>\n'
h+=' </tr>\n'
if image_orig!='' and himage!='':
h+=' <tr>\n'
h+=' <td valign="top"><b>Embedd original image into interactive report/paper:</b></td>\n'
h+=' <td valign="top"><i>\n'
h+=' '+himage.replace('<','&lt;').replace('>','&gt;')+'\n'
h+=' </i></td>\n'
h+=' </tr>\n'
h+=' </table>\n'
h+=' </td>\n'
h+=' </tr>\n'
h+='</table>\n'
h+='</div>\n'
h+='</center>\n'
return {'return':0, 'raw':raw, 'show_top':top, 'html':h, 'style':st}
##############################################################################
# replaying saved graph from CMD
def replay(i):
"""
Input: {
(repo_uoa) - repo UOA of s saved graph
data_uoa - data UOA of a saved graph
(id) - subgraph id
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
o=i.get('out','')
duoa=i['data_uoa']
ruoa=i.get('repo_uoa','')
ii={'action':'load',
'module_uoa':work['self_module_uid'],
'data_uoa':duoa,
'repo_uoa':ruoa}
rx=ck.access(ii)
if rx['return']>0: return rx
dd=rx['dict']
graphs=dd.get('graphs',[])
if len(graphs)==0:
return {'return':1, 'error':'no saved graphs found'}
igraph=-1
if len(graphs)==1:
igraph=0
elif len(graphs)>1:
gid=i.get('id','')
if gid=='':
if o=='con':
ck.out('Available subgraphs ID:')
for q in graphs:
ck.out(' '+q.get('id',''))
ck.out('')
return {'return':1, 'error':'more than one subgraph found - please, specify id'}
jgraph=0
for q in graphs:
if q.get('id','')==gid:
igraph=jgraph
break
jgraph+=1
if igraph==-1:
return {'return':1, 'error':'can\'t find subgraph'}
params=graphs[igraph].get('params',{})
# Replaying
params['action']='plot'
params['module_uoa']=work['self_module_uid']
return ck.access(params)